/** * Running one monitor: check → reconcile → persist. * * This is the Execution half of the monitoring loop. Everything it decides was * decided elsewhere — which monitors are due (`sweep.ts`), what a result means * (`keys.ts`, `builtin-monitors.ts`), and what should change (`reconcile.ts`) — * so this function's only job is to sequence those and write the outcome down. * * The checks themselves are injected rather than imported. A monitor run * otherwise reaches SSH, the module hook executor, and the audit subsystem, * which would make the loop untestable without a live fleet. */ import { eq } from 'drizzle-orm'; import type { DbClient } from '../../db/client'; import { type AlertSeverity, type Monitor, monitorRuns, monitors } from '../../db/schema'; import type { DriftCategory, DriftFinding } from '../audit/types'; import type { HealthCheckResult } from '../health-runner'; import { failingKeysFromFindings } from './builtin-monitors'; import { HEALTH_COVERAGE_CHECK, healthCoverageFailingKeys } from './health-coverage'; import type { ModuleCoverageInput } from './health-coverage'; import { HOOK_JAIL_CHECK, type HookJailState, hookJailFailingKeys } from './hook-jail'; import { type FailingKey, builtinMonitorKey, failingKeysFromHealthItems, moduleAlertKey, } from './keys'; import { reconcile } from './reconcile'; import { applyReconcileActions, confirmStillFailing, loadLiveAlerts } from './store'; export interface MonitorRunDeps { /** Run a module's health_check hook unattended. */ runModuleCheck(moduleId: string): Promise; /** Run one audit category. */ runBuiltinCheck(category: DriftCategory): Promise; /** Module roster for the health-coverage check. */ loadModuleCoverage(): ModuleCoverageInput[]; /** Recorded jail mode and current host, for the hook-jail self-monitor. */ loadJailState(): HookJailState; now(): Date; /** Grace window before a new alert may notify. */ graceMs: number; } export interface MonitorRunOutcomeSummary { monitorId: string; outcome: 'success' | 'error'; errorMessage?: string; failingKeys: FailingKey[]; createdIds: string[]; resolvedIds: string[]; } /** * What the check produced, before it becomes alert state. * * `outcome` is the load-bearing field: `error` means the check could not run, * which says nothing about any individual key and must never resolve anything. */ interface CheckProduct { outcome: 'success' | 'error'; failingKeys: FailingKey[]; errorMessage?: string; } async function runCheck( monitor: Monitor, deps: MonitorRunDeps, severity: AlertSeverity, ): Promise { if (monitor.kind === 'module_hook') { const result = await deps.runModuleCheck(monitor.target); // `error` is the hook failing to execute. `no-checks` means the module // declares no health_check hook — nothing ran, but nothing was expected to, // so it is a successful run with an empty failing set rather than a fault. if (result.status === 'error') { return { outcome: 'error', failingKeys: [], errorMessage: result.error }; } return { outcome: 'success', failingKeys: failingKeysFromHealthItems( monitor.target, result.checks, severity, result.artifactPaths, ), }; } // The coverage check reads local state only, so it has no failure mode worth // distinguishing from an empty result. if (monitor.target === HEALTH_COVERAGE_CHECK) { return { outcome: 'success', failingKeys: healthCoverageFailingKeys(deps.loadModuleCoverage()), }; } // Same shape: one local file read. This is the self-monitor for D8's third // state — a host that used to jail its hooks and has stopped. if (monitor.target === HOOK_JAIL_CHECK) { return { outcome: 'success', failingKeys: hookJailFailingKeys(deps.loadJailState(), severity), }; } const category = monitor.target as DriftCategory; try { const findings = await deps.runBuiltinCheck(category); return { outcome: 'success', failingKeys: failingKeysFromFindings(category, findings, severity), }; } catch (error) { return { outcome: 'error', failingKeys: [], errorMessage: error instanceof Error ? error.message : String(error), }; } } /** The key meaning "this monitor could not run". */ export function monitorLevelKeyFor(monitor: Monitor): string { return monitor.kind === 'module_hook' ? moduleAlertKey(monitor.target) : builtinMonitorKey(monitor.target); } /** * Run one monitor and apply its consequences. * * Records the run's outcome before anything else derived from it, so that a * crash between checking and reconciling leaves evidence of what happened * rather than a silent gap. */ export async function runOneMonitor( db: DbClient, monitor: Monitor, deps: MonitorRunDeps, ): Promise { const now = deps.now(); const product = await runCheck(monitor, deps, monitor.severity); db.insert(monitorRuns) .values({ monitorId: monitor.id, ranAt: now, outcome: product.outcome, errorMessage: product.errorMessage, }) .run(); const liveAlerts = loadLiveAlerts(db, monitor.id); const actions = reconcile({ liveAlerts, outcome: product.outcome, failingKeys: product.failingKeys, monitorLevelKey: monitorLevelKeyFor(monitor), errorMessage: product.errorMessage, monitorSeverity: monitor.severity, now, graceMs: deps.graceMs, }); const { createdIds, resolvedIds } = applyReconcileActions(db, actions, { monitorId: monitor.id, now, }); // A successful run that still reports a key confirms any alert waiting on // confirmation after un-suppression. Only a run that EXECUTED may do this — // an errored run has confirmed nothing. if (product.outcome === 'success') { const stillFailing = new Set(product.failingKeys.map((f) => f.key)); const confirmable = liveAlerts.filter((a) => stillFailing.has(a.key)).map((a) => a.id); confirmStillFailing(db, confirmable, now); } db.update(monitors).set({ lastRunAt: now }).where(eq(monitors.id, monitor.id)).run(); return { monitorId: monitor.id, outcome: product.outcome, errorMessage: product.errorMessage, failingKeys: product.failingKeys, createdIds, resolvedIds, }; }