/** * The parts of pause that exist so a pause cannot become a silent outage * (design D7), plus the alerting suppressor. * * Every check here is asserted in BOTH directions per Rule 7.6 — a gate nobody * has seen fail is not a gate, and "no paused module was reported" is exactly * the output a broken detector produces. */ import { describe, expect, test } from 'bun:test'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { type DbClient, createDbClient } from '../db/client'; import { type ModuleState, modules } from '../db/schema'; import { moduleAlertKey } from './alerting/keys'; import { findSuppressor } from './alerting/suppression'; import { checkPausedModules } from './fleet-checks'; import { describeMachineStopInfra, describePausedModule, listPausedModules, pausedAmong, } from './module-pause'; function makeDb(): DbClient { const dir = mkdtempSync(join(tmpdir(), 'celilo-pause-obs-')); process.env.CELILO_DB_PATH = join(dir, 'celilo.db'); return createDbClient({ path: process.env.CELILO_DB_PATH }); } function insert( db: DbClient, id: string, state: ModuleState, opts: { pausedAt?: Date; reason?: string } = {}, ): void { db.insert(modules) .values({ id, name: id, version: '1.0.0', state, manifestData: { id, name: id, version: '1.0.0' }, sourcePath: `/tmp/${id}`, pausedAt: state === 'PAUSED' ? (opts.pausedAt ?? new Date()) : null, pauseReason: state === 'PAUSED' ? (opts.reason ?? null) : null, }) .run(); } describe('system doctor reports ANY paused module (task 6.2 / 6.4)', () => { test('with nothing paused the check passes', () => { const db = makeDb(); insert(db, 'caddy', 'VERIFIED'); const finding = checkPausedModules(db); expect(finding.status).toBe('ok'); expect(finding.summary).toBe('nothing paused'); }); test('PROVE IT FAILS: one paused module is a doctor FAILURE naming it and its age', () => { const db = makeDb(); insert(db, 'caddy', 'VERIFIED'); insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000), reason: 'edge router swap', }); const finding = checkPausedModules(db); expect(finding.status).toBe('fail'); expect(finding.summary).toContain('greenwave'); // The DURATION is what separates a maintenance window from a forgotten one. expect(finding.detail.join('\n')).toContain('3d'); expect(finding.detail.join('\n')).toContain('edge router swap'); expect(finding.remediation).toContain('celilo module unpause'); }); test('there is no threshold — a pause taken seconds ago already fails', () => { // Decided at review: a pause is a degraded state, full stop. A threshold is // just something to tune until the detector stops firing. const db = makeDb(); insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date() }); expect(checkPausedModules(db).status).toBe('fail'); }); }); describe('listPausedModules / pausedAmong', () => { test('reads only PAUSED rows, and reads back the reason', () => { const db = makeDb(); insert(db, 'caddy', 'INSTALLED'); insert(db, 'greenwave', 'PAUSED', { reason: 'edge router swap' }); insert(db, 'authentik', 'ERROR'); const paused = listPausedModules(db); expect(paused.map((m) => m.id)).toEqual(['greenwave']); expect(paused[0].pauseReason).toBe('edge router swap'); }); test('an ERROR module is NOT paused — the two states are distinct', () => { // Worth pinning: ERROR is *pausable*, which is easy to misread as "errored // counts as paused" and would make the doctor check fire on every failure. const db = makeDb(); insert(db, 'authentik', 'ERROR'); expect(listPausedModules(db)).toEqual([]); }); test('pausedAmong filters a candidate set in one query', () => { const db = makeDb(); insert(db, 'caddy', 'PAUSED'); insert(db, 'authentik', 'INSTALLED'); expect(pausedAmong(db, ['caddy', 'authentik', 'absent'])).toEqual(new Set(['caddy'])); }); test('pausedAmong on an empty list does not query at all', () => { const db = makeDb(); expect(pausedAmong(db, [])).toEqual(new Set()); }); }); describe('the management-API warning is emitted, and NOT emitted (task 6.5 / 6.6)', () => { // `fleetWarnings()` in api/serve.ts is the production caller; this pins the // query + rendering it depends on. The inverse assertion is the one that // stops the warning becoming noise nobody reads. test('with nothing paused there is nothing to warn about', () => { const db = makeDb(); insert(db, 'caddy', 'VERIFIED'); expect(listPausedModules(db)).toHaveLength(0); }); test('with something paused the warning names the module AND its age', () => { const db = makeDb(); insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date(Date.now() - 5 * 60 * 60 * 1000), reason: 'edge router swap', }); const rendered = listPausedModules(db).map((m) => describePausedModule(m)); expect(rendered).toEqual(['greenwave (5h, "edge router swap")']); }); test('a pause with no reason still renders its age', () => { const db = makeDb(); insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date(Date.now() - 60 * 60 * 1000) }); expect(describePausedModule(listPausedModules(db)[0])).toBe('greenwave (1h)'); }); }); describe('a paused module is suppressed BY THE PAUSE, not anonymously (task 2.3)', () => { const topology = { moduleSystems: [], zoneProviders: [] }; test('PROVE IT FIRES: an unpaused module with nothing firing is NOT suppressed', () => { expect( findSuppressor({ key: moduleAlertKey('caddy'), firingKeys: new Set(), suppressible: true, modulesInDeployWindow: new Set(), pausedModules: new Set(), topology, }), ).toBeNull(); }); test('the same alert IS suppressed once the module is paused, attributed to the pause', () => { expect( findSuppressor({ key: moduleAlertKey('caddy'), firingKeys: new Set(), suppressible: true, modulesInDeployWindow: new Set(), pausedModules: new Set(['caddy']), topology, }), ).toEqual({ kind: 'paused', moduleId: 'caddy' }); }); test('a pause does not suppress OTHER modules', () => { expect( findSuppressor({ key: moduleAlertKey('authentik'), firingKeys: new Set(), suppressible: true, modulesInDeployWindow: new Set(), pausedModules: new Set(['caddy']), topology, }), ).toBeNull(); }); test('a self-monitor is never suppressed, even by a pause', () => { // A cascading failure must not silence the component reporting the cascade. expect( findSuppressor({ key: moduleAlertKey('caddy'), firingKeys: new Set(), suppressible: false, modulesInDeployWindow: new Set(), pausedModules: new Set(['caddy']), topology, }), ).toBeNull(); }); }); describe('--stop-infra acts only on celilo-provisioned infrastructure (design D2, revised)', () => { // The RULING is about ownership, not capability, and the distinction lives in // the message the operator reads — so the message is what gets pinned. // // celilo could SSH into a machine and stop something. It declines to, because // a machine-pool system is operator-pre-provisioned, may predate celilo, and // may run work celilo was never told about. Reporting "celilo cannot identify // the service unit" would imply a capability gap where the truth is that this // is not celilo's to stop. test('the machine-hosted report reads as not-applicable, not as a missing feature', () => { const report = describeMachineStopInfra('iot', 'homebridge'); expect(report).toContain('not applicable'); expect(report).toContain('celilo provisioned'); // Must NOT frame it as something celilo would do if only it could. expect(report).not.toMatch(/cannot determine|does not know which|unit/i); }); });