import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { eq } from 'drizzle-orm'; import type { DbClient } from '../../db/client'; import { type Monitor, alerts, monitorRuns, monitors } from '../../db/schema'; import { setupTestDatabaseAt } from '../../test-utils/database'; import { resetTestDbPath } from '../../test-utils/db-path'; import type { HealthCheckResult } from '../health-runner'; import { HOOK_JAIL_CHECK, type HookJailState } from './hook-jail'; import { moduleAlertKey, moduleCheckAlertKey } from './keys'; import { type MonitorRunDeps, runOneMonitor } from './run-monitor'; import { loadLiveAlerts, markUnsuppressed } from './store'; const NOW = new Date('2026-07-28T03:00:00Z'); const LATER = new Date('2026-07-28T03:15:00Z'); const MODULE = 'caddy'; const DISK = moduleCheckAlertKey(MODULE, 'disk-space'); const MODULE_KEY = moduleAlertKey(MODULE); const healthy: HealthCheckResult = { moduleId: MODULE, status: 'healthy', checks: [] }; const diskFailing: HealthCheckResult = { moduleId: MODULE, status: 'unhealthy', checks: [{ name: 'disk-space', status: 'fail', message: '/var 94% used' }], }; const unreachable: HealthCheckResult = { moduleId: MODULE, status: 'error', checks: [], error: 'ssh: connect to host caddy.dmz port 22: timed out', }; describe('runOneMonitor', () => { let dir: string; let db: DbClient; let monitor: Monitor; function deps(result: HealthCheckResult, now: Date = NOW): MonitorRunDeps { return { runModuleCheck: async () => result, runBuiltinCheck: async () => [], loadModuleCoverage: () => [], loadJailState: () => ({ record: undefined, host: 'test-host' }), now: () => now, graceMs: 60_000, }; } const liveKeys = () => loadLiveAlerts(db, monitor.id) .map((a) => a.key) .sort(); beforeEach(async () => { dir = mkdtempSync(join(tmpdir(), 'run-monitor-')); const dbPath = join(dir, 'celilo.db'); process.env.CELILO_DB_PATH = dbPath; db = await setupTestDatabaseAt(dbPath); db.insert(monitors) .values({ id: 'mon-1', kind: 'module_hook', target: MODULE, intervalMinutes: 15, severity: 'critical', }) .run(); monitor = db.select().from(monitors).where(eq(monitors.id, 'mon-1')).get() as Monitor; }); afterEach(() => { db.$client.close(); resetTestDbPath(); try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }); test('a failing check creates an alert and records a successful run', async () => { const summary = await runOneMonitor(db, monitor, deps(diskFailing)); expect(summary.outcome).toBe('success'); expect(liveKeys()).toEqual([DISK]); const runs = db.select().from(monitorRuns).all(); expect(runs).toHaveLength(1); expect(runs[0].outcome).toBe('success'); }); test('a clean check resolves what it no longer reports', async () => { await runOneMonitor(db, monitor, deps(diskFailing)); await runOneMonitor(db, monitor, deps(healthy, LATER)); expect(liveKeys()).toEqual([]); }); test('lastRunAt advances so the sweep will not immediately re-run it', async () => { await runOneMonitor(db, monitor, deps(diskFailing)); const row = db.select().from(monitors).where(eq(monitors.id, 'mon-1')).get(); expect(row?.lastRunAt).toEqual(NOW); }); // The end-to-end version of the guard. A problem is already firing when the // box drops off the network. describe('when the system becomes unreachable', () => { test('the firing alert is NOT resolved', async () => { await runOneMonitor(db, monitor, deps(diskFailing)); await runOneMonitor(db, monitor, deps(unreachable, LATER)); const disk = db.select().from(alerts).where(eq(alerts.key, DISK)).get(); expect(disk?.state).not.toBe('resolved'); expect(disk?.resolvedAt).toBeNull(); }); test('the monitor-level alert fires carrying the real error', async () => { await runOneMonitor(db, monitor, deps(diskFailing)); await runOneMonitor(db, monitor, deps(unreachable, LATER)); const moduleAlert = db.select().from(alerts).where(eq(alerts.key, MODULE_KEY)).get(); expect(moduleAlert).toBeDefined(); expect(moduleAlert?.message).toContain('timed out'); }); test('the run is recorded as errored, not as a clean pass', async () => { await runOneMonitor(db, monitor, deps(unreachable)); const run = db.select().from(monitorRuns).all()[0]; expect(run.outcome).toBe('error'); expect(run.errorMessage).toContain('timed out'); }); test('both alerts stand until a run actually executes again', async () => { await runOneMonitor(db, monitor, deps(diskFailing)); await runOneMonitor(db, monitor, deps(unreachable, LATER)); expect(liveKeys()).toEqual([DISK, MODULE_KEY].sort()); }); }); describe('recovery', () => { test('a successful run clears the monitor-level alert but keeps a real failure', async () => { await runOneMonitor(db, monitor, deps(diskFailing)); await runOneMonitor(db, monitor, deps(unreachable, LATER)); await runOneMonitor(db, monitor, deps(diskFailing, new Date('2026-07-28T03:30:00Z'))); expect(liveKeys()).toEqual([DISK]); }); test('a successful clean run clears everything the outage froze', async () => { await runOneMonitor(db, monitor, deps(diskFailing)); await runOneMonitor(db, monitor, deps(unreachable, LATER)); await runOneMonitor(db, monitor, deps(healthy, new Date('2026-07-28T03:30:00Z'))); expect(liveKeys()).toEqual([]); }); }); describe('confirmation after un-suppression', () => { test('a still-failing key clears awaitingConfirmation', async () => { await runOneMonitor(db, monitor, deps(diskFailing)); const [alert] = loadLiveAlerts(db, monitor.id); markUnsuppressed(db, alert.id, LATER); await runOneMonitor(db, monitor, deps(diskFailing, LATER)); const row = db.select().from(alerts).where(eq(alerts.id, alert.id)).get(); expect(row?.awaitingConfirmation).toBe(false); }); // An errored run confirms nothing — the alert must keep waiting rather // than becoming eligible to page on the strength of a run that never ran. test('an errored run does NOT clear awaitingConfirmation', async () => { await runOneMonitor(db, monitor, deps(diskFailing)); const [alert] = loadLiveAlerts(db, monitor.id); markUnsuppressed(db, alert.id, LATER); await runOneMonitor(db, monitor, deps(unreachable, LATER)); const row = db.select().from(alerts).where(eq(alerts.id, alert.id)).get(); expect(row?.awaitingConfirmation).toBe(true); }); }); test('a module with no health_check hook is a clean run, not a fault', async () => { const noChecks: HealthCheckResult = { moduleId: MODULE, status: 'no-checks', checks: [] }; const summary = await runOneMonitor(db, monitor, deps(noChecks)); expect(summary.outcome).toBe('success'); expect(liveKeys()).toEqual([]); }); describe('the hook-jail self-monitor (hook-process-boundary task 4.4)', () => { const HOST = 'celilo-mgr'; const regressed: HookJailState = { host: HOST, record: { mode: 'unjailed', backend: 'none', reason: 'bubblewrap is installed but could not build a namespace', host: HOST, recordedAt: NOW.toISOString(), lastJailed: { backend: 'bubblewrap', recordedAt: '2026-07-27T03:00:00.000Z' }, }, }; const healed: HookJailState = { host: HOST, record: { mode: 'jailed', backend: 'bubblewrap', host: HOST, recordedAt: LATER.toISOString(), }, }; let jailMonitor: Monitor; beforeEach(() => { db.insert(monitors) .values({ id: 'mon-jail', kind: 'builtin_check', target: HOOK_JAIL_CHECK, intervalMinutes: 15, severity: 'critical', suppressible: false, }) .run(); jailMonitor = db.select().from(monitors).where(eq(monitors.id, 'mon-jail')).get() as Monitor; }); const jailDeps = (state: HookJailState, now: Date = NOW): MonitorRunDeps => ({ ...deps(healthy, now), loadJailState: () => state, }); test('a jailed-to-unjailed transition fires an alert naming the host', async () => { const summary = await runOneMonitor(db, jailMonitor, jailDeps(regressed)); expect(summary.outcome).toBe('success'); const live = loadLiveAlerts(db, jailMonitor.id); expect(live.map((a) => a.key)).toEqual([`builtin:hook_jail/host:${HOST}`]); const row = db.select().from(alerts).where(eq(alerts.monitorId, jailMonitor.id)).get(); expect(row?.message).toContain(HOST); }); test('re-jailing resolves it by set difference', async () => { await runOneMonitor(db, jailMonitor, jailDeps(regressed)); await runOneMonitor(db, jailMonitor, jailDeps(healed, LATER)); expect(loadLiveAlerts(db, jailMonitor.id)).toEqual([]); }); }); });