/** * The INBOUND direction — aspects applied to systems that have just come into * existence (celilo#902). * * The outbound direction (`maybeRunAspectForTrigger`, one provider across the * whole fleet) is covered in `aspect-runner.test.ts`. Nothing there could have * caught this bug: the fan-out enumerates the fleet once, at the moment the * providing module deploys, and every assertion is about systems that already * existed at that instant. * * The runner is injected throughout, so these exercise the GATING — which * provider's aspect runs on which host, and why — without driving Ansible. */ import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { closeDb, getDb } from '../db/client'; import { runMigrations } from '../db/migrate'; import { type ModuleState, modules } from '../db/schema'; import type { BaseModuleAspect } from '../manifest/schema'; import { resetTestDbPath } from '../test-utils/db-path'; import { computeAspectScopeHash, recordAspectApproval, recordAspectConsent, } from './aspect-approvals'; import { type AspectRunResult, planAspectFanOut, reconcileAspectsForSystems, verifyAspectCoverage, } from './aspect-runner'; import type { AnsibleHostRecap } from './deploy-ansible'; import { upsertDeployedSystem } from './deployed-systems'; const dnsAspect: BaseModuleAspect = { ansible_role: 'dns-client-config', applicable_zones: ['dmz', 'app', 'secure', 'internal'], // Deliberately ONLY on_install: the inbound reconcile must run anyway // (design D2), and this is the manifest shape both shipping aspects carry. triggers: ['on_install'], }; /** Seed a provider module with an aspect, approved unless told otherwise. */ function seedProvider(opts: { moduleId: string; aspect?: BaseModuleAspect; state?: ModuleState; approval?: 'approved' | 'denied' | 'none'; }): void { const db = getDb(); const aspect = opts.aspect ?? dnsAspect; db.insert(modules) .values({ id: opts.moduleId, name: opts.moduleId, version: '1.0.0', state: opts.state ?? 'INSTALLED', manifestData: { id: opts.moduleId, name: opts.moduleId, version: '1.0.0', celilo_contract: '1.0', base_module_aspect: aspect, }, sourcePath: `/tmp/${opts.moduleId}`, }) .run(); const approval = opts.approval ?? 'approved'; if (approval === 'approved') { recordAspectApproval({ moduleId: opts.moduleId, version: '1.0.0', scopeHash: computeAspectScopeHash(aspect), approver: 'test', db, }); } else if (approval === 'denied') { recordAspectConsent({ moduleId: opts.moduleId, version: '1.0.0', scopeHash: computeAspectScopeHash(aspect), approver: 'test', consented: false, db, }); } } /** A module with a deployed system, used as a fan-out TARGET. */ function seedTargetSystem(opts: { moduleId: string; hostname: string; zone: 'dmz' | 'app' | 'secure' | 'internal'; ip: string; }): void { const db = getDb(); db.insert(modules) .values({ id: opts.moduleId, name: opts.moduleId, version: '1.0.0', state: 'INSTALLED', manifestData: { id: opts.moduleId, name: opts.moduleId, version: '1.0.0', celilo_contract: '1.0', }, sourcePath: `/tmp/${opts.moduleId}`, }) .run(); upsertDeployedSystem(db, opts.moduleId, { name: 'main', hostname: opts.hostname, ipv4Address: opts.ip, zone: opts.zone, infraType: 'container_service', }); } interface RunnerCall { moduleId: string; trigger: string; onlyHostnames?: string[]; } /** A stand-in runner that records what it was asked to do. */ function fakeRunner(calls: RunnerCall[], success = true) { return (async (args: { moduleId: string; options: { trigger: string; onlyHostnames?: string[] }; }): Promise => { calls.push({ moduleId: args.moduleId, trigger: args.options.trigger, onlyHostnames: args.options.onlyHostnames, }); return { success, output: '', error: success ? undefined : 'ansible said no', plan: { targetSystems: [], skipped: [] }, recap: [], }; }) as unknown as Parameters[0]['runner']; } describe('reconcileAspectsForSystems', () => { let dir: string; beforeEach(async () => { dir = mkdtempSync(join(tmpdir(), 'celilo-aspect-reconcile-test-')); process.env.CELILO_DB_PATH = join(dir, 'celilo.db'); await runMigrations(process.env.CELILO_DB_PATH); }); afterEach(() => { closeDb(); resetTestDbPath(); try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }); it('applies an approved aspect to a system in a covered zone', async () => { seedProvider({ moduleId: 'knot-unbound-internal' }); const calls: RunnerCall[] = []; const result = await reconcileAspectsForSystems({ systems: [{ hostname: 'caddy-int', zone: 'dmz' }], db: getDb(), runner: fakeRunner(calls), }); expect(calls).toHaveLength(1); expect(calls[0].moduleId).toBe('knot-unbound-internal'); expect(calls[0].onlyHostnames).toEqual(['caddy-int']); expect(result.failures).toHaveLength(0); }); it('runs an aspect that declares ONLY on_install — triggers do not gate this direction', async () => { // Design D2, and the whole reason this fix does not need a manifest change. // If this ever starts consulting `aspect.triggers`, every shipping aspect // would need `on_new_system_in_zone` added — which changes its scope hash, // invalidates the operator's approval, and raises a re-approval interview // on the live fleet as a side effect of a bug fix. seedProvider({ moduleId: 'knot-unbound-internal', aspect: dnsAspect }); expect(dnsAspect.triggers).toEqual(['on_install']); const calls: RunnerCall[] = []; await reconcileAspectsForSystems({ systems: [{ hostname: 'vpn', zone: 'app' }], db: getDb(), runner: fakeRunner(calls), }); expect(calls).toHaveLength(1); expect(calls[0].trigger).toBe('on_new_system_in_zone'); }); it('does not apply an aspect to a system outside its applicable_zones', async () => { seedProvider({ moduleId: 'knot-unbound-internal', aspect: { ...dnsAspect, applicable_zones: ['dmz'] }, }); const calls: RunnerCall[] = []; const result = await reconcileAspectsForSystems({ systems: [{ hostname: 'vpn', zone: 'app' }], db: getDb(), runner: fakeRunner(calls), }); expect(calls).toHaveLength(0); expect(result.outcomes[0].reason).toBe('no_covered_systems'); }); it('does not apply a DENIED aspect, and does not re-prompt', async () => { seedProvider({ moduleId: 'knot-unbound-internal', approval: 'denied' }); const calls: RunnerCall[] = []; const result = await reconcileAspectsForSystems({ systems: [{ hostname: 'caddy-int', zone: 'dmz' }], db: getDb(), runner: fakeRunner(calls), requestConsent: async () => { throw new Error('must not interview for an already-denied aspect'); }, }); expect(calls).toHaveLength(0); expect(result.outcomes[0].reason).toBe('denied'); }); it('skips a PAUSED provider — the documented escape hatch for a wedged aspect', async () => { // Design D4a. An inbound failure is fatal to the deploy, so an operator // whose non-essential aspect is wedging every deploy pauses its provider, // deploys, and unpauses. If this stopped working there would be no way out. seedProvider({ moduleId: 'knot-unbound-internal', state: 'PAUSED' }); const calls: RunnerCall[] = []; const result = await reconcileAspectsForSystems({ systems: [{ hostname: 'caddy-int', zone: 'dmz' }], db: getDb(), runner: fakeRunner(calls), }); expect(calls).toHaveLength(0); expect(result.outcomes[0].reason).toBe('paused'); expect(result.failures).toHaveLength(0); }); it('skips the deploying module’s own aspect — on_install already fans it out', async () => { seedProvider({ moduleId: 'knot-unbound-internal' }); const calls: RunnerCall[] = []; await reconcileAspectsForSystems({ systems: [{ hostname: 'dns-int', zone: 'dmz' }], db: getDb(), excludeModuleIds: ['knot-unbound-internal'], runner: fakeRunner(calls), }); expect(calls).toHaveLength(0); }); it('reports a failed aspect as a failure the caller can act on', async () => { seedProvider({ moduleId: 'knot-unbound-internal' }); const calls: RunnerCall[] = []; const result = await reconcileAspectsForSystems({ systems: [{ hostname: 'caddy-int', zone: 'dmz' }], db: getDb(), runner: fakeRunner(calls, false), }); expect(result.failures).toHaveLength(1); expect(result.failures[0].providerModuleId).toBe('knot-unbound-internal'); expect(result.failures[0].error).toBe('ansible said no'); }); it('does nothing when no systems were created', async () => { seedProvider({ moduleId: 'knot-unbound-internal' }); const calls: RunnerCall[] = []; const result = await reconcileAspectsForSystems({ systems: [], db: getDb(), runner: fakeRunner(calls), }); expect(calls).toHaveLength(0); expect(result.outcomes).toHaveLength(0); }); }); describe('planAspectFanOut onlyHostnames', () => { let dir: string; beforeEach(async () => { dir = mkdtempSync(join(tmpdir(), 'celilo-aspect-only-test-')); process.env.CELILO_DB_PATH = join(dir, 'celilo.db'); await runMigrations(process.env.CELILO_DB_PATH); }); afterEach(() => { closeDb(); resetTestDbPath(); try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }); it('narrows the plan to the named hosts', async () => { seedTargetSystem({ moduleId: 'caddy', hostname: 'caddy', zone: 'dmz', ip: '10.0.10.10' }); seedTargetSystem({ moduleId: 'authentik', hostname: 'auth', zone: 'app', ip: '10.0.20.10' }); const all = await planAspectFanOut(dnsAspect); expect(all.targetSystems.map((t) => t.hostname).sort()).toEqual(['auth', 'caddy']); const narrowed = await planAspectFanOut(dnsAspect, { onlyHostnames: ['auth'] }); expect(narrowed.targetSystems.map((t) => t.hostname)).toEqual(['auth']); }); it('NARROWS ONLY — a host outside applicable_zones is still not a target', async () => { // Otherwise an inbound reconcile could apply an aspect to a system whose // zone the operator never approved, which is the consent surface. seedTargetSystem({ moduleId: 'signal', hostname: 'signal', zone: 'app', ip: '10.0.20.90' }); const dmzOnly: BaseModuleAspect = { ...dnsAspect, applicable_zones: ['dmz'] }; const plan = await planAspectFanOut(dmzOnly, { onlyHostnames: ['signal'] }); expect(plan.targetSystems).toHaveLength(0); }); }); describe('verifyAspectCoverage', () => { let dir: string; beforeEach(async () => { dir = mkdtempSync(join(tmpdir(), 'celilo-aspect-coverage-test-')); process.env.CELILO_DB_PATH = join(dir, 'celilo.db'); await runMigrations(process.env.CELILO_DB_PATH); }); afterEach(() => { closeDb(); resetTestDbPath(); try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }); /** A runner that reports a fixed check-mode recap for the named host. */ function recapRunner(recap: AnsibleHostRecap[]) { return (async (): Promise => ({ success: true, output: '', plan: { targetSystems: [], skipped: [] }, recap, })) as unknown as Parameters[0]['runner']; } function seedFleet(): void { seedProvider({ moduleId: 'knot-unbound-internal' }); seedTargetSystem({ moduleId: 'caddy', hostname: 'caddy', zone: 'dmz', ip: '10.0.10.10' }); } it('reports a host with no changes and no skips as applied', async () => { seedFleet(); const findings = await verifyAspectCoverage({ db: getDb(), runner: recapRunner([ { host: 'caddy', ok: 3, changed: 0, unreachable: 0, failed: 0, skipped: 0 }, ]), }); expect(findings).toHaveLength(1); expect(findings[0].state).toBe('applied'); }); it('reports a host the role would change as missing', async () => { seedFleet(); const findings = await verifyAspectCoverage({ db: getDb(), runner: recapRunner([ { host: 'caddy', ok: 3, changed: 1, unreachable: 0, failed: 0, skipped: 0 }, ]), }); expect(findings[0].state).toBe('missing'); expect(findings[0].detail).toContain('celilo module deploy knot-unbound-internal'); }); it('reports SKIPPED tasks as unknown — never as applied', async () => { // The unsafe direction, and the reason this is not a boolean. A role of // command:/shell: tasks finishes a check run with changed=0 having never // been applied; calling that "applied" is a confidently clean answer about // an unconverged host — the same failure as the stored verdict this whole // approach exists to avoid (celilo#902 design D6). seedFleet(); const findings = await verifyAspectCoverage({ db: getDb(), runner: recapRunner([ { host: 'caddy', ok: 1, changed: 0, unreachable: 0, failed: 0, skipped: 2 }, ]), }); expect(findings[0].state).toBe('unknown'); expect(findings[0].state).not.toBe('applied'); expect(findings[0].detail).toContain('check mode cannot evaluate'); }); it('reports a host with NO recap line as unknown, not as applied', async () => { // An empty recap means nothing was measured. Reading it as success is how a // verification turns into the thing it was meant to replace. seedFleet(); const findings = await verifyAspectCoverage({ db: getDb(), runner: recapRunner([]) }); expect(findings[0].state).toBe('unknown'); expect(findings[0].detail).toContain('no recap'); }); it('reports an unreachable host distinctly from a measured one', async () => { seedFleet(); const findings = await verifyAspectCoverage({ db: getDb(), runner: recapRunner([ { host: 'caddy', ok: 0, changed: 0, unreachable: 1, failed: 0, skipped: 0 }, ]), }); expect(findings[0].state).toBe('unreachable'); }); it('reports a PAUSED provider’s entitled systems, so pausing cannot go unnoticed', async () => { seedProvider({ moduleId: 'knot-unbound-internal', state: 'PAUSED' }); seedTargetSystem({ moduleId: 'caddy', hostname: 'caddy', zone: 'dmz', ip: '10.0.10.10' }); const findings = await verifyAspectCoverage({ db: getDb(), runner: recapRunner([]), }); expect(findings).toHaveLength(1); expect(findings[0].state).toBe('unknown'); expect(findings[0].detail).toContain('PAUSED'); }); it('does not verify an unapproved aspect, and never raises an interview', async () => { seedProvider({ moduleId: 'knot-unbound-internal', approval: 'none' }); seedTargetSystem({ moduleId: 'caddy', hostname: 'caddy', zone: 'dmz', ip: '10.0.10.10' }); const findings = await verifyAspectCoverage({ db: getDb(), runner: recapRunner([]) }); expect(findings).toHaveLength(0); }); });