/** * `celilo monitor` — what celilo is watching, and running a check on demand. * * Thin adapter (Rule 10.5): parse, delegate, format. The scheduling decisions * live in services/alerting/, and this command composes the real health runner * and audit checks into the injectable deps `runOneMonitor` expects. */ import { hostname } from 'node:os'; import { defineEvents, openBus } from '@celilo/event-bus'; import { getEventBusPath } from '../../config/paths'; import { getDb } from '../../db/client'; import type { MonitorKind } from '../../db/schema'; import { readJailMode } from '../../hooks/jail'; import { isSchedulableBuiltin, runBuiltinCheckForMonitor, } from '../../services/alerting/builtin-source'; import { loadModuleCoverage } from '../../services/alerting/coverage-source'; import { renderMonitorRunMessage } from '../../services/alerting/format'; import { loadModuleHealthCadences } from '../../services/alerting/health-cadence'; import { HEALTH_COVERAGE_CHECK } from '../../services/alerting/health-coverage'; import { HOOK_JAIL_CHECK } from '../../services/alerting/hook-jail'; import { createMonitor, ensureSweepSubscriber, findMonitorByTarget, listMonitors, setMonitorEnabled, updateMonitorInterval, } from '../../services/alerting/monitors'; import { listPolicies } from '../../services/alerting/people'; import { runOneMonitor } from '../../services/alerting/run-monitor'; import { promoteReadyAlerts } from '../../services/alerting/store'; import type { DriftCategory } from '../../services/audit/types'; import { MONITOR_INTERVAL_FLOOR_MINUTES, cadenceSchema, formatCadence, parseCadence, } from '../../services/cadence'; import { runModuleHealthCheck } from '../../services/health-runner'; import type { CommandResult } from '../types'; const NO_SCHEMAS = defineEvents({}); /** Grace window before a newly-fired alert may notify. */ const DEFAULT_GRACE_MS = 60_000; function buildDeps() { const db = getDb(); return { runModuleCheck: (moduleId: string) => runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }), runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category, db), loadModuleCoverage: () => loadModuleCoverage(db), loadJailState: () => ({ record: readJailMode(), host: hostname() }), now: () => new Date(), graceMs: DEFAULT_GRACE_MS, }; } function handleList(): CommandResult { const rows = listMonitors(getDb()); if (rows.length === 0) { console.log('\nNo monitors configured.\n'); console.log('A module declaring hooks.health_check.interval gets one on deploy,'); console.log('or add one directly: celilo monitor add --interval 15m\n'); return { success: true, message: 'No monitors configured' }; } // The POLICY column is the answer to "why did nothing page me". A monitor // with no policy is one whose alerts reach nobody, and until this column // existed there was no way to see that from the CLI at all — `assign` // reported success and nothing anywhere reflected the result (#481). const policies = new Map(listPolicies(getDb()).map((p) => [p.id, p.name])); const policyOf = (id: string | null) => id ? (policies.get(id) ?? '(deleted policy)') : '— pages nobody'; // A `module_hook` row's stored interval is not what the sweep uses, so // printing it would be a confident lie. Resolve the same way the sweep does. const cadences = loadModuleHealthCadences(getDb()); const everyOf = (monitor: (typeof rows)[number]): string => { if (monitor.kind !== 'module_hook') return `${monitor.intervalMinutes}m`; const cadence = cadences.get(monitor.target)?.cadence ?? null; return cadence === null ? '—' : formatCadence(cadence); }; const width = Math.max(6, ...rows.map((r) => r.target.length)); const policyWidth = Math.max(6, ...rows.map((r) => policyOf(r.escalationPolicyId).length)); console.log(''); console.log( `${'TARGET'.padEnd(width)} ${'KIND'.padEnd(14)} ${'EVERY'.padEnd(8)} ${'POLICY'.padEnd(policyWidth)} STATE`, ); for (const monitor of rows) { const every = everyOf(monitor); const state = monitor.kind === 'module_hook' ? every === 'manual' || every === '—' ? 'not watched' : 'watched' : monitor.enabled ? 'enabled' : 'disabled'; const suffix = monitor.lastRunAt ? '' : ' (never run)'; console.log( `${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(8)} ${policyOf(monitor.escalationPolicyId).padEnd(policyWidth)} ${state}${suffix}`, ); } console.log(''); return { success: true, message: `${rows.length} monitor(s)` }; } function handleAdd(args: string[], flags: Record): CommandResult { const target = args[0]; if (!target) { return { success: false, error: 'Usage: celilo monitor add [--interval 15m]' }; } const db = getDb(); // A target naming an audit category is a built-in check; anything else is a // module's health_check hook. `isSchedulableBuiltin` is checked explicitly // because not every category is snake_case — `backups` is one word, and the // underscore heuristic alone would file it as a module hook against a module // that does not exist. const kind: MonitorKind = target === HEALTH_COVERAGE_CHECK || target === HOOK_JAIL_CHECK || isSchedulableBuiltin(target) || target.includes('_') ? 'builtin_check' : 'module_hook'; // Checked BEFORE the already-exists check, so an operator reaching for the // cadence knob is told where it moved rather than told the row exists. A // module's cadence does not live on its row, so accepting `--interval` here // would store a number nothing reads — refuse rather than ignore. if (kind === 'module_hook' && typeof flags.interval === 'string') { return { success: false, error: moduleCadenceRedirect(target, '--interval') }; } if (findMonitorByTarget(db, target)) { return { success: false, error: kind === 'module_hook' ? `A monitor for "${target}" already exists — a deploy creates one for every module with a health_check hook.\n\nTo change how often it runs: celilo module config set ${target} health_check_interval ` : `A monitor for "${target}" already exists.\n\nTo change how often it runs: celilo monitor set-interval ${target} `, }; } const interval = typeof flags.interval === 'string' ? flags.interval : '15m'; const invalid = validateMonitorCadence(interval); if (invalid) return { success: false, error: invalid }; const cadence = parseCadence(interval); const intervalMinutes = cadence !== null && cadence !== 'manual' ? cadence.minutes : 0; // The hook-jail check is a SELF-monitor: it watches celilo's own confinement // of module hooks, and the alerting spec marks monitors watching the system // itself unsuppressible. That is a property of the target, not an operator // choice — losing the jail during a broad outage is exactly when no ancestor // alert may silence it. const suppressible = target === HOOK_JAIL_CHECK ? false : undefined; createMonitor(db, { kind, target, intervalMinutes, suppressible }); // Creating the first monitor is also what switches the sweep on. Registering // here rather than at install means a celilo with no monitors carries no // subscriber and does no periodic work. const bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS }); try { ensureSweepSubscriber(bus); } finally { bus.close(); } return { success: true, message: kind === 'module_hook' ? `Watching ${target}. How often is per-module policy: celilo module config set ${target} health_check_interval ` : `Monitoring ${target} every ${interval} (sweep runs on timer.tick.5m)`, }; } async function handleRun(args: string[]): Promise { const target = args[0]; if (!target) return { success: false, error: 'Usage: celilo monitor run ' }; const db = getDb(); const monitor = findMonitorByTarget(db, target); if (!monitor) { return { success: false, error: `No monitor for "${target}".\n\nRun "celilo monitor list" to see configured monitors.`, }; } const summary = await runOneMonitor(db, monitor, buildDeps()); promoteReadyAlerts(db, new Date()); return { success: true, message: renderMonitorRunMessage(target, summary) }; } function handleToggle(args: string[], enabled: boolean): CommandResult { const target = args[0]; const verb = enabled ? 'enable' : 'disable'; if (!target) return { success: false, error: `Usage: celilo monitor ${verb} ` }; const db = getDb(); const monitor = findMonitorByTarget(db, target); if (!monitor) return { success: false, error: `No monitor for "${target}".` }; if (monitor.kind === 'module_hook') { return { success: false, error: moduleCadenceRedirect(target, `monitor ${verb}`) }; } setMonitorEnabled(db, monitor.id, enabled, new Date()); return { success: true, message: `Monitor for ${target} ${enabled ? 'enabled' : 'disabled'}` }; } /** * Two ways to change one module's cadence would disagree about what `module * status` shows, so the fleet-level commands refuse a module target and name * the per-module one. The targets are disjoint in practice — an operator has * either a module or an audit check in hand — and an explicit error teaches * better than silence. */ function moduleCadenceRedirect(target: string, what: string): string { return ( `${what} does not apply to "${target}" — it is a module, and a module's cadence is per-module policy rather than a monitor setting. Its monitor row is created by the deploy.\n\n` + ` celilo module config set ${target} health_check_interval 15m # watch it every 15 minutes\n` + ` celilo module config set ${target} health_check_interval manual # stop watching it\n` + ` celilo module config unset ${target} health_check_interval # follow the manifest again` ); } /** The floor comes from the alerting sweep's own tick, never a written-down number. */ function validateMonitorCadence(value: string): string | null { const result = cadenceSchema({ floorMinutes: MONITOR_INTERVAL_FLOOR_MINUTES }).safeParse(value); if (result.success) return null; return `Invalid interval "${value}".\n\n${result.error.issues.map((i) => i.message).join('\n')}`; } /** * `celilo monitor set-interval ` — re-cadence a built-in check * without removing and recreating it, which would orphan its alert history. */ function handleSetInterval(args: string[]): CommandResult { const [target, cadence] = args; if (!target || !cadence) { return { success: false, error: 'Usage: celilo monitor set-interval ' }; } const db = getDb(); const monitor = findMonitorByTarget(db, target); if (!monitor) return { success: false, error: `No monitor for "${target}".` }; if (monitor.kind === 'module_hook') { return { success: false, error: moduleCadenceRedirect(target, 'monitor set-interval') }; } const invalid = validateMonitorCadence(cadence); if (invalid) return { success: false, error: invalid }; const parsed = parseCadence(cadence); if (parsed === null || parsed === 'manual') { return { success: false, error: `A built-in check has no "manual" — disable it instead: celilo monitor disable ${target}`, }; } updateMonitorInterval(db, monitor.id, parsed.minutes); return { success: true, message: `${target} now runs every ${formatCadence(parsed)}` }; } export async function handleMonitor( subcommand: string | undefined, args: string[], flags: Record = {}, ): Promise { switch (subcommand) { case undefined: case 'list': return handleList(); case 'add': return handleAdd(args, flags); case 'run': return handleRun(args); case 'enable': return handleToggle(args, true); case 'disable': return handleToggle(args, false); case 'set-interval': return handleSetInterval(args); default: return { success: false, error: `Unknown monitor subcommand: ${subcommand}\n\nUse: list, add, run, set-interval, enable, disable`, }; } }