/** * Alert persistence — applying reconciler decisions to the alerts table. * * Kept separate from `reconcile.ts` on purpose: the decisions are pure and * exhaustively tested without a database, and this layer only has to get the * writes right. The one piece of judgement here is `activeKey`, which must * move in lockstep with `state` or the "one live alert per key" index either * rejects legitimate re-fires or permits duplicates. */ import { randomUUID } from 'node:crypto'; import { and, eq, inArray, isNotNull, lte } from 'drizzle-orm'; import type { DbClient } from '../../db/client'; import { type Alert, type AlertSeverity, alerts } from '../../db/schema'; import type { LiveAlert, ReconcileAction } from './reconcile'; /** Alert states that are still "live" — i.e. not resolved. */ const LIVE_STATES = ['pending', 'firing', 'acked', 'suppressed'] as const; /** Load the alerts a monitor currently owns that have not resolved. */ export function loadLiveAlerts(db: DbClient, monitorId: string): LiveAlert[] { return db .select({ id: alerts.id, key: alerts.key }) .from(alerts) .where(and(eq(alerts.monitorId, monitorId), isNotNull(alerts.activeKey))) .all(); } export interface ApplyContext { monitorId: string; now: Date; } /** * Apply reconciler actions. * * Returns the ids of alerts created and resolved, so the caller can drive * notification (a resolve fans out an all-clear only to routes that were * actually told about the alert). */ export function applyReconcileActions( db: DbClient, actions: ReconcileAction[], context: ApplyContext, ): { createdIds: string[]; resolvedIds: string[] } { const createdIds: string[] = []; const resolvedIds: string[] = []; for (const action of actions) { if (action.type === 'create') { const id = randomUUID(); db.insert(alerts) .values({ id, key: action.key, // Mirrors `key` while live; NULLed on resolve. This is what the // unique index keys off — SQLite treats NULLs as distinct, so any // number of resolved rows may share a key while at most one live // row may hold it. activeKey: action.key, monitorId: context.monitorId, state: 'pending', severity: action.severity, firstFiredAt: context.now, lastSeenAt: context.now, graceUntil: action.graceUntil, escalationStep: 0, message: action.message, details: action.details, }) .run(); createdIds.push(id); continue; } if (action.type === 'refresh') { // Severity and message can change between runs (a warning becoming a // failure, a percentage climbing); the alert's identity does not. db.update(alerts) .set({ lastSeenAt: context.now, severity: action.severity, message: action.message, details: action.details, }) .where(eq(alerts.id, action.alertId)) .run(); continue; } db.update(alerts) .set({ state: 'resolved', activeKey: null, resolvedAt: context.now }) .where(eq(alerts.id, action.alertId)) .run(); resolvedIds.push(action.alertId); } return { createdIds, resolvedIds }; } /** * Record that an alert has become suppressed by an ancestor alert or a deploy * window. Idempotent: re-suppressing an already-suppressed alert is a no-op * rather than resetting its bookkeeping. */ export function markSuppressed( db: DbClient, alertId: string, by: { alertId?: string; windowId?: string }, ): void { db.update(alerts) .set({ state: 'suppressed', suppressedByAlertId: by.alertId ?? null, suppressedByWindowId: by.windowId ?? null, }) .where(eq(alerts.id, alertId)) .run(); } /** * Lift suppression. * * Two things happen together, and both matter: * * - `awaitingConfirmation` is set, so the alert does NOT notify until a * subsequent run confirms it is still failing. When a machine comes back, * its modules almost always came back with it; paging for them would * recreate the storm suppression just prevented. * - the escalation clock is restarted from now. Running it from the original * fire time would fire every step whose delay had nominally elapsed during * the suppression, all at once. */ export function markUnsuppressed(db: DbClient, alertId: string, now: Date): void { db.update(alerts) .set({ state: 'firing', suppressedByAlertId: null, suppressedByWindowId: null, unsuppressedAt: now, awaitingConfirmation: true, escalationStep: 0, nextEscalationAt: now, }) .where(eq(alerts.id, alertId)) .run(); } /** * Clear `awaitingConfirmation` for alerts a SUCCESSFUL run has just confirmed * are still failing. Only a run that executed may clear it — that is the same * asymmetry the reconciler rests on, applied to a different column. */ export function confirmStillFailing(db: DbClient, alertIds: string[], now: Date): void { if (alertIds.length === 0) return; db.update(alerts) .set({ awaitingConfirmation: false, nextEscalationAt: now }) .where(inArray(alerts.id, alertIds)) .run(); } /** * Promote `pending` alerts whose grace window has elapsed to `firing`. * * Without this an alert never leaves `pending`, because nothing else writes * that transition — `reconcile` only creates and resolves. Escalation would * still behave correctly (it compares `graceUntil` directly), but every alert * would read as `pending` forever in `celilo alerts` and in the health column, * which is a lie about the system's state even though nothing downstream acts * on it. * * Suppressed alerts are left alone: they are already accounted for, and moving * them to `firing` would lose the record of what explains them. */ export function promoteReadyAlerts(db: DbClient, now: Date): number { const ready = db .select({ id: alerts.id }) .from(alerts) .where(and(eq(alerts.state, 'pending'), lte(alerts.graceUntil, now))) .all(); if (ready.length === 0) return 0; db.update(alerts) .set({ state: 'firing' }) .where( inArray( alerts.id, ready.map((r) => r.id), ), ) .run(); return ready.length; } /** * Record that an escalation step was taken. * * Called for a delivered message AND for one deferred by quiet hours, because * the step is what advances, not the message (D13). Without this the sweep * would re-decide the same step every five minutes and never reach the * secondary. */ export function recordStepTaken( db: DbClient, alertId: string, taken: { stepIndex: number; nextStepDueAt: Date | null }, ): void { db.update(alerts) .set({ escalationStep: taken.stepIndex + 1, nextEscalationAt: taken.nextStepDueAt }) .where(eq(alerts.id, alertId)) .run(); } /** * Hold a message until someone's quiet hours end. * * Keeps the EARLIEST pending deferral rather than the latest: if a second step * defers behind a longer window, the operator still hears about it as soon as * anyone is reachable. */ export function deferNotification( db: DbClient, alertId: string, deferral: { routeId: string; until: Date }, ): void { const current = db .select({ until: alerts.deferredUntil }) .from(alerts) .where(eq(alerts.id, alertId)) .get(); if (current?.until && current.until <= deferral.until) return; db.update(alerts) .set({ deferredUntil: deferral.until, deferredRouteId: deferral.routeId }) .where(eq(alerts.id, alertId)) .run(); } /** Alerts whose deferral window has ended and are owed a message. */ export function dueDeferrals(db: DbClient, now: Date): Alert[] { return db .select() .from(alerts) .where(and(inArray(alerts.state, [...LIVE_STATES]), lte(alerts.deferredUntil, now))) .all(); } export function clearDeferral(db: DbClient, alertId: string): void { db.update(alerts) .set({ deferredUntil: null, deferredRouteId: null }) .where(eq(alerts.id, alertId)) .run(); } /** Every live alert, for the notify sweep and the health column. */ export function loadAllLiveAlerts(db: DbClient): Alert[] { return db .select() .from(alerts) .where(inArray(alerts.state, [...LIVE_STATES])) .all(); } export interface AlertSummary { key: string; severity: AlertSeverity; suppressed: boolean; } /** Live alerts grouped by module id, for the `celilo module list` column. */ export function summariseByModule(liveAlerts: Alert[]): Map { const byModule = new Map(); for (const alert of liveAlerts) { if (!alert.key.startsWith('module:')) continue; const rest = alert.key.slice('module:'.length); const moduleId = rest.split('/')[0]; if (!moduleId) continue; const list = byModule.get(moduleId) ?? []; list.push({ key: alert.key, severity: alert.severity, suppressed: alert.state === 'suppressed', }); byModule.set(moduleId, list); } return byModule; }