/** * Monitor CRUD. * * A monitor binds a thing to check to a severity and an escalation policy. * Operators address them by their target (`caddy`, `machines_reachable`) rather * than by id — per CLAUDE.md, a UUID never reaches the operator. * * ⚠️ `intervalMinutes` and `enabled` are meaningful for `builtin_check` rows * ONLY. A built-in check targets a fleet-level audit category with no module * and no manifest, so nothing suggests its cadence and the row legitimately IS * the config. A `module_hook` row targets a module whose manifest MAY suggest * one, so its cadence and whether it is watched at all resolve at read time * from `health_check_interval` — see [[services/alerting/health-cadence.ts]]. * Two columns whose meaning depends on `kind` is a smell, named here rather * than discovered in review (design.md D8); the alternatives are a resolved * value cached on the row, which rots, or splitting the table, which needs a * synthetic monitor identity for `alerts.monitorId` and friends. */ import { randomUUID } from 'node:crypto'; import { and, eq, isNotNull } from 'drizzle-orm'; import type { DbClient } from '../../db/client'; import { type AlertSeverity, type Monitor, type MonitorKind, alerts, monitors, } from '../../db/schema'; import { ALERTING_SWEEP_PATTERN, MONITOR_INTERVAL_FLOOR_MINUTES, parseCadence } from '../cadence'; export interface CreateMonitorInput { kind: MonitorKind; target: string; /** `builtin_check` only. Not consulted for `module_hook` — see the file header. */ intervalMinutes: number; severity?: AlertSeverity; suppressible?: boolean; escalationPolicyId?: string | null; } export function listMonitors(db: DbClient): Monitor[] { return db.select().from(monitors).all(); } export function findMonitor(db: DbClient, kind: MonitorKind, target: string): Monitor | undefined { return db .select() .from(monitors) .where(and(eq(monitors.kind, kind), eq(monitors.target, target))) .get(); } /** * Find a monitor by target alone. * * Targets are unique in practice — a module id and an audit category cannot * collide, since audit categories are a fixed snake_case set and module ids are * kebab-case. Resolving by target alone is what lets the operator type * `celilo monitor run caddy` without also naming the kind. */ export function findMonitorByTarget(db: DbClient, target: string): Monitor | undefined { return db.select().from(monitors).where(eq(monitors.target, target)).get(); } export function createMonitor(db: DbClient, input: CreateMonitorInput): Monitor { const id = randomUUID(); db.insert(monitors) .values({ id, kind: input.kind, target: input.target, intervalMinutes: input.intervalMinutes, severity: input.severity ?? 'critical', suppressible: input.suppressible ?? true, escalationPolicyId: input.escalationPolicyId ?? null, }) .run(); return db.select().from(monitors).where(eq(monitors.id, id)).get() as Monitor; } /** * Ensure a `module_hook` monitor row exists for a module that declares a * `health_check` hook. * * The row is no longer where the cadence lives — it carries severity, the * escalation policy and `lastRunAt`, and whether it runs at all is resolved * from the module's effective cadence at sweep time. So it is created for any * module with the hook, not only for one whose manifest happens to suggest an * interval: an operator can now set a cadence on a module whose author never * named one, and without a row there would be nothing to carry its last run. * * The stored `intervalMinutes` is the manifest's suggestion where there is one, * and is NOT read back for this kind. It is written so that rolling back to a * release which does read it behaves as it did before. * * Idempotent: a second call is a no-op, which is what keeps deploys from * duplicating rows. It is no longer what protects an operator's setting — that * lives in `module_configs` now and cannot be overwritten from here at all. */ export function ensureMonitorForModule( db: DbClient, moduleId: string, suggestedInterval: string | undefined, ): Monitor | null { if (findMonitor(db, 'module_hook', moduleId)) return null; const suggested = suggestedInterval ? parseCadence(suggestedInterval) : null; const intervalMinutes = suggested !== null && suggested !== 'manual' ? suggested.minutes : MONITOR_INTERVAL_FLOOR_MINUTES; return createMonitor(db, { kind: 'module_hook', target: moduleId, intervalMinutes }); } /** * Change a `builtin_check` monitor's cadence in place. * * In place, rather than remove-and-recreate, because the monitor id owns the * alert history: recreating it would orphan every live alert it raised. */ export function updateMonitorInterval(db: DbClient, monitorId: string, minutes: number): void { db.update(monitors).set({ intervalMinutes: minutes }).where(eq(monitors.id, monitorId)).run(); } /** * Enable or disable a monitor. * * Disabling resolves its alerts rather than leaving them hanging: a disabled * monitor will never report on them again, so anything still live would be * frozen forever with no way to clear. */ export function setMonitorEnabled( db: DbClient, monitorId: string, enabled: boolean, now: Date, ): void { db.update(monitors).set({ enabled }).where(eq(monitors.id, monitorId)).run(); if (!enabled) resolveMonitorAlerts(db, monitorId, now); } /** Resolve every live alert owned by a monitor. */ export function resolveMonitorAlerts(db: DbClient, monitorId: string, now: Date): number { const live = db .select({ id: alerts.id }) .from(alerts) .where(and(eq(alerts.monitorId, monitorId), isNotNull(alerts.activeKey))) .all(); for (const row of live) { db.update(alerts) .set({ state: 'resolved', activeKey: null, resolvedAt: now }) .where(eq(alerts.id, row.id)) .run(); } return live.length; } /** * The bus subscriber that makes alerting run by itself. * * Registered as an ordinary subscriber with a shell handler, exactly like a * module subscription — the dispatcher spawns `celilo alerts sweep` on every * five-minute tick. No new scheduling mechanism: `timer.tick.5m` already * exists, already survives restarts, and already has retry and dedup. * * Idempotent: `bus.subscribe` upserts by name, so calling this on every * monitor creation is safe and means switching on the first monitor is also * what switches on the sweep. */ export const ALERTING_SWEEP_SUBSCRIBER = 'celilo-alerting-sweep'; // The tick itself lives in services/cadence.ts, next to the floor derived from // it — a sweep whose tick and whose finest servable cadence are stated in two // files is the pair that drifts. export { ALERTING_SWEEP_PATTERN }; export interface SubscriberRegistrar { subscribe(options: { name: string; pattern: string; handler: string; registeredBy?: string; }): unknown; } export const ALERTING_POLL_SUBSCRIBER = 'celilo-alerting-inbound'; export function ensureSweepSubscriber(bus: SubscriberRegistrar): void { bus.subscribe({ name: ALERTING_SWEEP_SUBSCRIBER, pattern: ALERTING_SWEEP_PATTERN, handler: 'celilo alerts sweep', registeredBy: 'celilo-alerting', }); } /** * Read inbound replies on the same five-minute tick. * * Five minutes is coarse for an ack — the design wants seconds — but the bus * timer menu is deliberately fixed, and a coarse ack that works beats a * bespoke scheduler that has to be supervised. An operator wanting faster * turnaround runs `celilo alerts poll`, and a tighter loop can be added later * without changing anything else. */ export function ensureInboundSubscriber(bus: SubscriberRegistrar): void { bus.subscribe({ name: ALERTING_POLL_SUBSCRIBER, pattern: ALERTING_SWEEP_PATTERN, handler: 'celilo alerts poll', registeredBy: 'celilo-alerting', }); } /** A live alert about to be deleted along with the monitor that owns it. */ export interface DroppedAlert { key: string; message: string; } /** * Delete a monitor, and report the live alerts that go with it. * * The alerts are DELETED, not resolved. `alerts.monitorId` is * `on delete cascade`, so the row goes the moment the monitor does. This used * to call `resolveMonitorAlerts` first; that write was unreachable — the very * next statement dropped the same rows, and nothing read them in between, so * the resolved state existed for the duration of one statement. Deleted rather * than restored, because resolving properly would need the alert rows to * outlive their monitor, which the FK forbids and which no reader wants: * `resolvedAt` is only ever read as a liveness predicate (`ack.ts`), never * reported. Do not put the call back without changing the FK first. * * What IS worth keeping is what was lost, which is why the live alerts are * returned. A monitor dropped while holding a firing alert takes a real * failure and the coverage of it away together, and a caller that only names * the monitor cannot tell an operator what stopped being watched. */ export function deleteMonitor(db: DbClient, monitorId: string): DroppedAlert[] { const dropped = db .select({ key: alerts.key, message: alerts.message }) .from(alerts) .where(and(eq(alerts.monitorId, monitorId), isNotNull(alerts.activeKey))) .all(); db.delete(monitors).where(eq(monitors.id, monitorId)).run(); return dropped; } /** * Drop the `module_hook` monitor belonging to a module that is going away. * * Called from the removal path rather than expressed as a foreign key, because * `monitors.target` cannot carry one: it holds a module id for `module_hook` * and an audit check name for `builtin_check`, and SQLite has no conditional * reference. Splitting the table to get the constraint would need a synthetic * monitor identity for `alerts.monitorId` — the trade the file header already * weighs and declines. * * Deleting cascades the monitor's alerts away (`alerts.monitorId` is * `on delete cascade`), which is what releases anything they were suppressing. * They are deleted, not resolved — see `deleteMonitor`. Returns what was live * so the caller can say what stopped being watched, or null if the module had * no monitor. */ export function deleteMonitorForModule(db: DbClient, moduleId: string): DroppedAlert[] | null { const monitor = findMonitor(db, 'module_hook', moduleId); if (!monitor) return null; return deleteMonitor(db, monitor.id); }