import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { eq } from 'drizzle-orm'; import type { DbClient } from '../../db/client'; import { type Alert, type Monitor, alerts, modules, monitors } from '../../db/schema'; import { setupTestDatabaseAt } from '../../test-utils/database'; import { resetTestDbPath } from '../../test-utils/db-path'; import type { HealthCheckResult } from '../health-runner'; import { deleteModuleConfig, upsertModuleConfig } from '../module-config'; import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, reconcileModuleWatchState } from './health-cadence'; import { moduleCheckAlertKey } from './keys'; import type { MonitorRunDeps } from './run-monitor'; import { type SuppressionTopology, machineAlertKey } from './suppression'; import { type SweepDeps, runSweep } from './sweep-runner'; const NOW = new Date('2026-07-28T12:00:00Z'); const later = (minutes: number) => new Date(NOW.getTime() + minutes * 60_000); const MODULE = 'homebridge'; const PORT_CHECK = moduleCheckAlertKey(MODULE, 'port'); const failing: HealthCheckResult = { moduleId: MODULE, status: 'unhealthy', checks: [{ name: 'port', status: 'fail', message: 'port 8581 closed' }], }; const healthy: HealthCheckResult = { moduleId: MODULE, status: 'healthy', checks: [] }; const TOPOLOGY: SuppressionTopology = { moduleSystems: [{ moduleId: MODULE, hostname: 'iot', zone: 'internal', infraType: 'machine' }], zoneProviders: [], }; describe('runSweep', () => { let dir: string; let db: DbClient; let monitor: Monitor; function monitorDeps(result: HealthCheckResult, now: Date): MonitorRunDeps { return { runModuleCheck: async () => result, runBuiltinCheck: async () => [], loadModuleCoverage: () => [], loadJailState: () => ({ record: undefined, host: 'test-host' }), now: () => now, graceMs: 60_000, }; } function deps(over: Partial = {}, result = failing, now = NOW): SweepDeps { return { monitorDeps: monitorDeps(result, now), loadTopology: () => TOPOLOGY, loadDeployWindowModules: () => new Set(), loadPausedModules: () => new Set(), isSuppressible: () => true, // No routes configured: the sweep must still run everything else. notifyDepsFor: () => null, now: () => now, ...over, }; } const liveAlerts = (): Alert[] => db.select().from(alerts).all(); const currentMonitors = () => db.select().from(monitors).all(); beforeEach(async () => { dir = mkdtempSync(join(tmpdir(), 'sweep-')); const dbPath = join(dir, 'celilo.db'); process.env.CELILO_DB_PATH = dbPath; db = await setupTestDatabaseAt(dbPath); // The module row is load-bearing now: a `module_hook` monitor's cadence // resolves from the module's manifest and config, not from its own row. db.insert(modules) .values({ id: MODULE, name: MODULE, sourcePath: dir, version: '1.0.0', state: 'INSTALLED', manifestData: { id: MODULE, hooks: { health_check: { script: './h.ts', interval: '15m' } }, }, }) .run(); db.insert(monitors) .values({ id: 'mon-1', kind: 'module_hook', target: MODULE, intervalMinutes: 15, severity: 'critical', }) .run(); monitor = db.select().from(monitors).where(eq(monitors.id, 'mon-1')).get() as Monitor; }); afterEach(() => { db.$client.close(); resetTestDbPath(); try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }); test('a never-run monitor is due and its failure becomes an alert', async () => { const report = await runSweep(db, currentMonitors(), deps()); expect(report.monitorsRun).toBe(1); expect(liveAlerts().map((a) => a.key)).toEqual([PORT_CHECK]); }); // Inside the grace window an alert is recorded but not yet firing, so a // check that clears immediately never pages. test('a fresh alert stays pending until its grace window elapses', async () => { const report = await runSweep(db, currentMonitors(), deps()); expect(report.promoted).toBe(0); expect(liveAlerts()[0].state).toBe('pending'); }); test('a later sweep promotes it to firing', async () => { await runSweep(db, currentMonitors(), deps()); const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20))); expect(report.promoted).toBe(1); expect(liveAlerts()[0].state).toBe('firing'); }); test('a monitor within its interval is not re-run', async () => { await runSweep(db, currentMonitors(), deps()); // 5 minutes later, well inside the 15-minute interval. const report = await runSweep(db, currentMonitors(), deps({}, failing, later(5))); expect(report.monitorsRun).toBe(0); }); // `enabled` and `intervalMinutes` are builtin_check-only now (design.md D8). // A future reader that reintroduces the dependency on the row fails here // rather than silently regressing to write-time resolution. test("a module monitor's stored enabled flag is not consulted", async () => { db.update(monitors).set({ enabled: false }).where(eq(monitors.id, 'mon-1')).run(); const report = await runSweep(db, currentMonitors(), deps()); expect(report.monitorsRun).toBe(1); }); test("a module monitor's stored interval is not consulted", async () => { // The row says a full day; the manifest says 15m, which is what wins. db.update(monitors).set({ intervalMinutes: 1440 }).where(eq(monitors.id, 'mon-1')).run(); await runSweep(db, currentMonitors(), deps()); const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20))); expect(report.monitorsRun).toBe(1); }); test('a module whose effective cadence is manual is never run, and its alerts resolve', async () => { await runSweep(db, currentMonitors(), deps()); expect(liveAlerts()).not.toEqual([]); upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual'); reconcileModuleWatchState(db, MODULE, later(20)); const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20))); expect(report.monitorsRun).toBe(0); expect(liveAlerts().every((a) => a.state === 'resolved')).toBe(true); }); test('unsetting the override resumes watching', async () => { upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual'); expect((await runSweep(db, currentMonitors(), deps())).monitorsRun).toBe(0); deleteModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY); const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20))); expect(report.monitorsRun).toBe(1); }); test('recovery resolves the alert on a later sweep', async () => { await runSweep(db, currentMonitors(), deps()); await runSweep(db, currentMonitors(), deps({}, healthy, later(20))); expect(liveAlerts().every((a) => a.state === 'resolved')).toBe(true); }); // The whole point of step 3 running after step 1: the machine alert may not // exist yet when the module check fails. describe('suppression is applied across the sweep, not during it', () => { beforeEach(() => { // A SEPARATE monitor owns the machine alert. Attaching it to the module's // monitor would have it resolved by set difference on the very first // sweep — that monitor does not report the machine key, and absence from // a successful run means resolved. Correct behaviour, wrong fixture. db.insert(monitors) .values({ id: 'mon-machines', kind: 'builtin_check', target: 'machines_reachable', intervalMinutes: 5, severity: 'critical', enabled: false, }) .run(); db.insert(alerts) .values({ id: 'machine-alert', key: machineAlertKey('iot'), activeKey: machineAlertKey('iot'), monitorId: 'mon-machines', state: 'firing', severity: 'critical', graceUntil: NOW, message: 'iot unreachable', }) .run(); }); test('a module alert is suppressed by its machine being down', async () => { // Suppression lands on the FIRST sweep: step 3 runs after every monitor // has reported, so the machine alert is already visible. const report = await runSweep(db, currentMonitors(), deps()); expect(report.suppressed).toBeGreaterThanOrEqual(1); const moduleAlert = db.select().from(alerts).where(eq(alerts.key, PORT_CHECK)).get(); expect(moduleAlert?.state).toBe('suppressed'); expect(moduleAlert?.suppressedByAlertId).toBe(machineAlertKey('iot')); }); // Lifting suppression must NOT page immediately — it waits for a run that // confirms the problem outlived its cause. test('un-suppression sets awaitingConfirmation rather than notifying', async () => { await runSweep(db, currentMonitors(), deps()); // The machine recovers. db.update(alerts) .set({ state: 'resolved', activeKey: null }) .where(eq(alerts.id, 'machine-alert')) .run(); const report = await runSweep(db, currentMonitors(), deps({}, failing, later(40))); expect(report.unsuppressed).toBe(1); const moduleAlert = db.select().from(alerts).where(eq(alerts.key, PORT_CHECK)).get(); expect(moduleAlert?.awaitingConfirmation).toBe(true); expect(moduleAlert?.state).toBe('firing'); }); test('an unsuppressible alert is never suppressed', async () => { await runSweep(db, currentMonitors(), deps({ isSuppressible: () => false })); const report = await runSweep( db, currentMonitors(), deps({ isSuppressible: () => false }, failing, later(20)), ); expect(report.suppressed).toBe(0); }); }); // The sweep is the fleet's only heartbeat. One broken module must not stop // every other alert from being evaluated. test('a monitor that throws is counted, and the sweep continues', async () => { const exploding: MonitorRunDeps = { ...monitorDeps(failing, NOW), runModuleCheck: async () => { throw new Error('hook executor blew up'); }, }; const report = await runSweep(db, currentMonitors(), deps({ monitorDeps: exploding })); expect(report.monitorsErrored).toBe(1); }); test('a sweep with no monitors does nothing and does not throw', async () => { db.delete(monitors).run(); const report = await runSweep(db, [], deps()); expect(report).toMatchObject({ monitorsRun: 0, promoted: 0, notified: 0 }); }); test('lastRunAt advances so the next sweep respects the interval', async () => { await runSweep(db, currentMonitors(), deps()); expect(db.select().from(monitors).get()?.lastRunAt).toEqual(NOW); expect(monitor.lastRunAt).toBeNull(); }); // A delivery that never happened must be distinguishable from one that was // never needed. Both used to render as `notified: 0` and nothing else, which // is how a firing-but-undelivered alert became undebuggable (#450). describe('undelivered alerts are accounted for, not silently dropped', () => { test('an alert with no escalation policy is counted, not skipped in silence', async () => { // `notifyDepsFor` returning null IS "nobody is configured to be told" — // the default in every other test here, which is why this went unnoticed. const report = await runSweep(db, currentMonitors(), deps()); expect(liveAlerts()).toHaveLength(1); expect(report.notified).toBe(0); expect(report.noPolicy).toEqual([{ alertKey: PORT_CHECK, monitor: MODULE }]); }); // A count told the operator something was unroutable but not WHICH thing, // so the remedy printed next to it could not be aimed anywhere (#481). test('the unroutable alert is named, along with the monitor to assign to', async () => { const report = await runSweep(db, currentMonitors(), deps()); expect(report.noPolicy[0].alertKey).toBe(PORT_CHECK); expect(report.noPolicy[0].monitor).toBe(MODULE); }); test('escalation declining to notify records WHICH reason', async () => { // A fresh alert is inside its grace window, so escalation declines with // `within_grace` — a real skip reason reached through the real code path // rather than a stubbed outcome. const notifyDeps = { steps: [{ stepIndex: 0, routeId: 'route-1', delayMinutes: 0 }], routes: new Map([['route-1', { id: 'route-1', severityFloor: 'warning', enabled: true }]]), routeDetails: new Map([ [ 'route-1', { id: 'route-1', personId: 'p1', address: '+15550000000', canAck: false } as never, ], ]), quietHoursByPerson: new Map(), bypassQuietHours: false, transportFor: () => { throw new Error('transport must not be reached for a skipped delivery'); }, mintToken: () => 'tok', now: NOW, } as never; const report = await runSweep( db, currentMonitors(), deps({ notifyDepsFor: () => notifyDeps }), ); expect(report.notified).toBe(0); expect(report.noPolicy).toEqual([]); // The alert is named, not just counted — an operator asking "why was I // not paged" is asking about a specific alert (#450). expect(report.skipped).toEqual([ { alertKey: 'module:homebridge/check:port', reason: 'within_grace' }, ]); }); test('a transport that cannot be loaded records the error, not just a count', async () => { // The transport is resolved lazily INSIDE the send, so a capability that // will not load never reaches the transport's own logs. If the sweep does // not carry the message, it exists nowhere. const notifyDeps = (now: Date) => ({ steps: [{ stepIndex: 0, routeId: 'route-1', delayMinutes: 0 }], routes: new Map([ ['route-1', { id: 'route-1', severityFloor: 'warning', enabled: true }], ]), routeDetails: new Map([ [ 'route-1', { id: 'route-1', personId: 'p1', address: '+15550000000', canAck: false } as never, ], ]), quietHoursByPerson: new Map(), bypassQuietHours: false, transportFor: () => { throw new Error('does not provide the notification capability'); }, mintToken: () => 'tok', now, }) as never; // First sweep creates the alert; the second is past the grace window, so // escalation actually reaches the transport. await runSweep(db, currentMonitors(), deps()); const at = later(20); const report = await runSweep( db, currentMonitors(), deps({ notifyDepsFor: () => notifyDeps(at) }, failing, at), ); expect(report.notified).toBe(0); expect(report.failed).toBe(1); expect(report.failures).toHaveLength(1); expect(report.failures[0]).toContain(PORT_CHECK); expect(report.failures[0]).toContain('does not provide the notification capability'); }); }); });