/** * Unit tests for pause planning + execution. * * The planner is pure, so the interesting cases (ordering, refusals, walking * through already-done members) need no database at all. Execution is tested * against fake deps — the point is the state machine, not Proxmox. */ import { describe, expect, test } from 'bun:test'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { eq } from 'drizzle-orm'; import { type DbClient, createDbClient } from '../db/client'; import { ipAllocations, moduleConfigs, modules, secrets } from '../db/schema'; import type { ModuleState } from '../db/schema'; import type { ModuleManifest } from '../manifest/schema'; import { type ModuleSnapshot, type PauseDeps, type PausePlan, PauseRefusedError, actedOn, executePause, executeUnpause, formatPausedDuration, planPause, planUnpause, } from './module-pause'; function makeManifest( id: string, opts: { provides?: string[]; requires?: string[]; optional?: string[] } = {}, ): ModuleManifest { return { id, name: id, version: '1.0.0', celilo_contract: '1.0', provides: { capabilities: (opts.provides ?? []).map((name) => ({ name, version: '1.0.0', data: {}, functions: [], })), }, requires: { capabilities: (opts.requires ?? []).map((name) => ({ name, version: '1.0.0' })) }, optional: opts.optional ? { capabilities: opts.optional.map((name) => ({ name, version: '1.0.0' })) } : undefined, } as unknown as ModuleManifest; } function snap( id: string, state: ModuleState, opts: { provides?: string[]; requires?: string[]; optional?: string[] } = {}, ): ModuleSnapshot { return { id, state, pausedAt: state === 'PAUSED' ? new Date('2026-08-01T00:00:00Z') : null, pauseReason: state === 'PAUSED' ? 'swapping the edge router' : null, manifest: makeManifest(id, opts), }; } /** * The real shape of the greenwave → axon problem: a firewall provider with a * chain of consumers, one of which reaches it only via `optional`. * * greenwave ──provides firewall──> caddy (requires) ──provides public_web──> website * technitium (OPTIONAL dhcp_server) */ function fleet(states: Partial> = {}): ModuleSnapshot[] { const state = (id: string, fallback: ModuleState = 'INSTALLED') => states[id] ?? fallback; return [ snap('greenwave', state('greenwave'), { provides: ['firewall', 'dhcp_server'] }), snap('caddy', state('caddy'), { provides: ['public_web'], requires: ['firewall'] }), snap('website', state('website'), { requires: ['public_web'] }), snap('technitium', state('technitium'), { optional: ['dhcp_server'] }), snap('unrelated', state('unrelated')), ]; } describe('planPause — ordering', () => { test('a plain pause is single-module, whatever depends on it', () => { const plan = planPause({ moduleId: 'greenwave', fleet: fleet(), cascade: false }); expect(plan.steps.map((s) => s.moduleId)).toEqual(['greenwave']); }); test('cascade pause covers the transitive consumers and pauses them FIRST', () => { const plan = planPause({ moduleId: 'greenwave', fleet: fleet(), cascade: true }); const order = plan.steps.map((s) => s.moduleId); expect(new Set(order)).toEqual(new Set(['greenwave', 'caddy', 'website', 'technitium'])); expect(order).not.toContain('unrelated'); // Consumers before the provider they depend on — the ordering IS the // feature, so assert relative position rather than membership. expect(order.indexOf('website')).toBeLessThan(order.indexOf('caddy')); expect(order.indexOf('caddy')).toBeLessThan(order.indexOf('greenwave')); expect(order.indexOf('technitium')).toBeLessThan(order.indexOf('greenwave')); }); test('cascade unpause runs the same set in the opposite order — providers first', () => { const paused = fleet({ greenwave: 'PAUSED', caddy: 'PAUSED', website: 'PAUSED', technitium: 'PAUSED', }); const order = planUnpause({ moduleId: 'greenwave', fleet: paused, cascade: true }).steps.map( (s) => s.moduleId, ); expect(order.indexOf('greenwave')).toBeLessThan(order.indexOf('caddy')); expect(order.indexOf('caddy')).toBeLessThan(order.indexOf('website')); }); test('an `optional` consumer is in the cascade — the graph counts it as an edge', () => { const plan = planPause({ moduleId: 'greenwave', fleet: fleet(), cascade: true }); expect(plan.steps.map((s) => s.moduleId)).toContain('technitium'); }); }); describe('planPause — legal source states (task 3.7)', () => { test.each([['INSTALLED'], ['VERIFIED'], ['ERROR']] as const)( '%s is pausable', (state: ModuleState) => { const plan = planPause({ moduleId: 'greenwave', fleet: fleet({ greenwave: state }), cascade: false, }); expect(plan.steps[0]).toEqual({ moduleId: 'greenwave', disposition: 'act' }); }, ); test('a never-deployed module is refused, and the message says why', () => { expect(() => planPause({ moduleId: 'greenwave', fleet: fleet({ greenwave: 'IMPORTED' }), cascade: false }), ).toThrow(/never been deployed/); }); test('a module mid-transition is refused with a DIFFERENT message', () => { expect(() => planPause({ moduleId: 'greenwave', fleet: fleet({ greenwave: 'DEPLOYING' }), cascade: false, }), ).toThrow(/strand/); }); test('an in-flight operation refuses the pause and names the operation', () => { expect(() => planPause({ moduleId: 'greenwave', fleet: fleet(), cascade: false, inFlight: new Map([['greenwave', 'backup of greenwave (pid 4242)']]), }), ).toThrow(/backup of greenwave \(pid 4242\)/); }); test('a cascade is refused WHOLE when any member cannot be paused', () => { // caddy is mid-deploy. Pausing the rest and stopping would leave the // operator to work out which half happened. expect(() => planPause({ moduleId: 'greenwave', fleet: fleet({ caddy: 'DEPLOYING' }), cascade: true }), ).toThrow(PauseRefusedError); }); test('an unknown module is refused rather than silently planning nothing', () => { expect(() => planPause({ moduleId: 'nope', fleet: fleet(), cascade: false })).toThrow( /Module not found/, ); }); }); describe('a cascade walks THROUGH members already in the target condition (task 4.7)', () => { // The failure the operator specifically called out: `unpause --cascade caddy` // when caddy is already unpaused must still reach the modules beyond it. // A cascade that halted at the first already-done member would leave the // consumers paused forever, and a half-finished cascade unresumable. test('unpause --cascade on an ALREADY-UNPAUSED module still reaches its consumers', () => { const plan = planUnpause({ moduleId: 'caddy', fleet: fleet({ caddy: 'INSTALLED', website: 'PAUSED' }), cascade: true, }); const ids = plan.steps.map((s) => s.moduleId); expect(ids).toContain('caddy'); expect(ids).toContain('website'); expect(plan.steps.find((s) => s.moduleId === 'caddy')?.disposition).toBe('skip_already'); expect(plan.steps.find((s) => s.moduleId === 'website')?.disposition).toBe('act'); expect(actedOn(plan)).toEqual(['website']); }); test('pause --cascade over a partly paused set plans only the outstanding work', () => { const plan = planPause({ moduleId: 'greenwave', fleet: fleet({ website: 'PAUSED', technitium: 'PAUSED' }), cascade: true, }); expect(actedOn(plan).sort()).toEqual(['caddy', 'greenwave']); // Still PRESENT in the plan — the report tells the operator what was // already done rather than pretending it was not in scope. expect(plan.steps.map((s) => s.moduleId)).toContain('website'); }); test('an already-paused member does not trip the source-state refusal', () => { // PAUSED is not in PAUSABLE_STATES, so a naive validator would refuse the // whole cascade the second time it ran — i.e. exactly when resuming. expect(() => planPause({ moduleId: 'greenwave', fleet: fleet({ caddy: 'PAUSED' }), cascade: true }), ).not.toThrow(); }); }); // --------------------------------------------------------------------------- // Execution // --------------------------------------------------------------------------- /** * Execution runs against a REAL isolated SQLite database rather than a fake * drizzle chain. A hand-rolled fake cannot honour a `where` clause, so it * silently answered the wrong row for the read-before-redeploy that preserves * the original `pausedAt` — the fake would have hidden precisely the bug that * read exists to prevent. Everything else (deploy, bus, Proxmox) stays injected. */ interface FakeState { unsubscribed: string[]; resubscribed: string[]; redeployed: string[]; operations: Array<{ moduleId: string; kind: string; outcome: 'completed' | 'failed' }>; infraStopped: string[]; } function makeDb(): DbClient { const dir = mkdtempSync(join(tmpdir(), 'celilo-pause-')); process.env.CELILO_DB_PATH = join(dir, 'celilo.db'); return createDbClient({ path: process.env.CELILO_DB_PATH }); } function seed(db: DbClient, initial: Record): void { for (const [id, state] of Object.entries(initial)) { db.insert(modules) .values({ id, name: id, version: '1.0.0', state, manifestData: makeManifest(id) as unknown as Record, sourcePath: `/tmp/${id}`, pausedAt: state === 'PAUSED' ? new Date('2026-08-01T00:00:00Z') : null, pauseReason: state === 'PAUSED' ? 'original reason' : null, }) .run(); } } function readRow(db: DbClient, id: string) { return db.select().from(modules).where(eq(modules.id, id)).get(); } function fakeDeps( initial: Record, overrides: { failRedeployOf?: string } = {}, ): { deps: PauseDeps; state: FakeState; db: DbClient } { const db = makeDb(); seed(db, initial); const state: FakeState = { unsubscribed: [], resubscribed: [], redeployed: [], operations: [], infraStopped: [], }; const deps: PauseDeps = { db, unsubscribe: (id) => state.unsubscribed.push(id), resubscribe: (id) => state.resubscribed.push(id), redeploy: async (id) => { state.redeployed.push(id); if (overrides.failRedeployOf === id) { // A real failed deploy leaves `state` somewhere else entirely — often // ERROR. Model that, so the test proves the executor RESTORES PAUSED // rather than merely never having left it. db.update(modules).set({ state: 'ERROR' }).where(eq(modules.id, id)).run(); return { success: false, error: 'bad config' }; } db.update(modules).set({ state: 'INSTALLED' }).where(eq(modules.id, id)).run(); return { success: true }; }, stopInfrastructure: async (id) => { state.infraStopped.push(id); return { stopped: true, detail: `stopped ${id}` }; }, startOperation: (moduleId, kind) => { state.operations.push({ moduleId, kind, outcome: 'completed' }); return `${moduleId}:${kind}`; }, completeOperation: () => {}, failOperation: (operationId) => { const entry = state.operations.find((o) => `${o.moduleId}:${o.kind}` === operationId); if (entry) entry.outcome = 'failed'; }, now: () => new Date('2026-08-12T00:00:00Z'), log: () => {}, }; return { deps, state, db }; } function planFor(action: 'pause' | 'unpause', ids: string[], stopInfra = false): PausePlan { return { action, requested: ids[0], cascade: ids.length > 1, stopInfra, steps: ids.map((id) => ({ moduleId: id, disposition: 'act' as const })), }; } describe('executePause', () => { test('pauses, quiesces, and records the reason', async () => { const { deps, state, db } = fakeDeps({ greenwave: 'INSTALLED' }); const report = await executePause(planFor('pause', ['greenwave']), deps, 'edge router swap'); expect(report.success).toBe(true); expect(readRow(db, 'greenwave')?.state).toBe('PAUSED'); expect(readRow(db, 'greenwave')?.pauseReason).toBe('edge router swap'); expect(state.unsubscribed).toEqual(['greenwave']); }); test('does NOT stop infrastructure by default (design D2)', async () => { const { deps, state } = fakeDeps({ caddy: 'INSTALLED' }); await executePause(planFor('pause', ['caddy']), deps, null); // Pausing caddy to swap the FIREWALL must not take every website down. expect(state.infraStopped).toEqual([]); }); test('--stop-infra stops it', async () => { const { deps, state } = fakeDeps({ caddy: 'INSTALLED' }); await executePause(planFor('pause', ['caddy'], true), deps, null); expect(state.infraStopped).toEqual(['caddy']); }); test('re-pausing preserves the ORIGINAL pausedAt and reason (task 3.5)', async () => { const { deps, db } = fakeDeps({ greenwave: 'PAUSED' }); const plan: PausePlan = { action: 'pause', requested: 'greenwave', cascade: false, stopInfra: false, steps: [{ moduleId: 'greenwave', disposition: 'skip_already', note: 'already paused' }], }; const report = await executePause(plan, deps, 'a NEW reason'); expect(report.success).toBe(true); // The age must keep measuring the real outage, not reset on every retry. expect(readRow(db, 'greenwave')?.pausedAt).toEqual(new Date('2026-08-01T00:00:00Z')); expect(readRow(db, 'greenwave')?.pauseReason).toBe('original reason'); }); }); describe('executeUnpause', () => { test('redeploys, clears the pause, and re-arms the subscriptions', async () => { const { deps, state, db } = fakeDeps({ greenwave: 'PAUSED' }); const report = await executeUnpause(planFor('unpause', ['greenwave']), deps); expect(report.success).toBe(true); expect(state.redeployed).toEqual(['greenwave']); expect(readRow(db, 'greenwave')?.state).toBe('INSTALLED'); expect(readRow(db, 'greenwave')?.pausedAt).toBeNull(); // Deploy does not register subscriptions — without this the module comes // back deployed but permanently deaf. expect(state.resubscribed).toEqual(['greenwave']); }); test('a failed redeploy leaves the module PAUSED and quiesced (task 3.6)', async () => { const { deps, state, db } = fakeDeps({ caddy: 'PAUSED' }, { failRedeployOf: 'caddy' }); const report = await executeUnpause(planFor('unpause', ['caddy']), deps); expect(report.success).toBe(false); // The fake deploy moved it to ERROR; the executor must put it back. expect(readRow(db, 'caddy')?.state).toBe('PAUSED'); expect(readRow(db, 'caddy')?.pausedAt).toEqual(new Date('2026-08-01T00:00:00Z')); expect(state.unsubscribed).toContain('caddy'); expect(state.resubscribed).toEqual([]); expect(state.operations.find((o) => o.kind === 'unpause')?.outcome).toBe('failed'); }); test('a failed provider stops the cascade and leaves consumers paused, not mis-bound', async () => { const { deps, state, db } = fakeDeps( { greenwave: 'PAUSED', caddy: 'PAUSED', website: 'PAUSED' }, { failRedeployOf: 'greenwave' }, ); const report = await executeUnpause( planFor('unpause', ['greenwave', 'caddy', 'website']), deps, ); expect(report.success).toBe(false); // Redeploying a consumer against a provider that is not there is the exact // mis-binding this design exists to prevent. expect(state.redeployed).toEqual(['greenwave']); expect(readRow(db, 'caddy')?.state).toBe('PAUSED'); expect(readRow(db, 'website')?.state).toBe('PAUSED'); expect(report.outcomes.filter((o) => o.result === 'skipped')).toHaveLength(2); }); test('an already-unpaused member is skipped WITHOUT a redeploy (task 4.8)', async () => { const { deps, state, db } = fakeDeps({ caddy: 'INSTALLED', website: 'PAUSED' }); const plan: PausePlan = { action: 'unpause', requested: 'caddy', cascade: true, stopInfra: false, steps: [ { moduleId: 'caddy', disposition: 'skip_already', note: 'not paused' }, { moduleId: 'website', disposition: 'act' }, ], }; const report = await executeUnpause(plan, deps); expect(report.success).toBe(true); expect(state.redeployed).toEqual(['website']); expect(readRow(db, 'website')?.state).toBe('INSTALLED'); }); }); describe('formatPausedDuration', () => { const now = new Date('2026-08-12T12:00:00Z'); test.each([ [new Date('2026-08-12T11:59:30Z'), 'just now'], [new Date('2026-08-12T11:15:00Z'), '45m'], [new Date('2026-08-12T09:00:00Z'), '3h'], [new Date('2026-08-09T12:00:00Z'), '3d'], ])('%s → %s', (pausedAt, expected) => { expect(formatPausedDuration(pausedAt, now)).toBe(expected); }); test('a null timestamp reads as unknown rather than as zero', () => { // Rendering "just now" for a missing timestamp would make an old pause look // fresh, which is the one direction that matters. expect(formatPausedDuration(null, now)).toBe('unknown'); }); }); describe('a pause/unpause round trip preserves everything (task 1.4)', () => { // Pausing is NOT a partial uninstall. If a round trip lost config, secrets, or // the IPAM allocation, the provider swap would come back with a different IP // and the whole exercise would be worse than the removal it replaced. test('config, secret and IPAM/VMID rows are byte-identical before and after', async () => { const { deps, db } = fakeDeps({ caddy: 'INSTALLED' }); db.insert(moduleConfigs) .values({ moduleId: 'caddy', key: 'domain', value: 'example.test', valueJson: '"example.test"', }) .run(); db.insert(secrets) .values({ moduleId: 'caddy', name: 'api_key', encryptedValue: 'ciphertext', iv: 'iv-value', authTag: 'tag-value', }) .run(); db.insert(ipAllocations) .values({ moduleId: 'caddy', vmid: 231, containerIp: '10.0.20.31/24', zone: 'dmz' }) .run(); const before = { configs: db.select().from(moduleConfigs).where(eq(moduleConfigs.moduleId, 'caddy')).all(), secrets: db.select().from(secrets).where(eq(secrets.moduleId, 'caddy')).all(), ipam: db.select().from(ipAllocations).where(eq(ipAllocations.moduleId, 'caddy')).all(), }; await executePause(planFor('pause', ['caddy']), deps, 'edge router swap'); expect(readRow(db, 'caddy')?.state).toBe('PAUSED'); await executeUnpause(planFor('unpause', ['caddy']), deps); expect(readRow(db, 'caddy')?.state).toBe('INSTALLED'); expect( db.select().from(moduleConfigs).where(eq(moduleConfigs.moduleId, 'caddy')).all(), ).toEqual(before.configs); expect(db.select().from(secrets).where(eq(secrets.moduleId, 'caddy')).all()).toEqual( before.secrets, ); // The allocation in particular: releasing it would hand the address to the // next module and make the unpause land somewhere else entirely. expect( db.select().from(ipAllocations).where(eq(ipAllocations.moduleId, 'caddy')).all(), ).toEqual(before.ipam); }); test('the pause itself does not run on_uninstall', async () => { // Pausing is not a partial uninstall: withdrawing the module's cross-module // state is exactly what must NOT happen, or the consumers it registered for // would be torn down by a pause meant to protect them. const { deps, state } = fakeDeps({ caddy: 'INSTALLED' }); await executePause(planFor('pause', ['caddy']), deps, null); expect(state.operations.map((o) => o.kind)).toEqual(['pause']); expect(state.redeployed).toEqual([]); }); }); describe('a cascade skips undeployed members instead of refusing (found by e2e)', () => { // Regression: `pause --cascade greenwave` refused outright because // `technitium` was IMPORTED — swept in from the manifest graph, never // deployed. An undeployed module is not bound to the provider and has nothing // to quiesce, so blocking on it wedges exactly the migration this feature // exists to enable: one stray imported module stops the swap. test('an IMPORTED consumer is skipped, and the rest of the cascade proceeds', () => { const plan = planPause({ moduleId: 'greenwave', fleet: fleet({ technitium: 'IMPORTED' }), cascade: true, }); const technitium = plan.steps.find((s) => s.moduleId === 'technitium'); expect(technitium?.disposition).toBe('skip_undeployed'); expect(actedOn(plan).sort()).toEqual(['caddy', 'greenwave', 'website']); }); test.each([['IMPORTED'], ['VALIDATED'], ['CONFIGURED']] as const)( 'a %s consumer does not block the cascade', (state: ModuleState) => { expect(() => planPause({ moduleId: 'greenwave', fleet: fleet({ technitium: state }), cascade: true }), ).not.toThrow(); }, ); test('but naming an undeployed module DIRECTLY is still refused', () => { // The operator asked for something that cannot happen; that is worth saying. expect(() => planPause({ moduleId: 'technitium', fleet: fleet({ technitium: 'IMPORTED' }), cascade: false, }), ).toThrow(/never been deployed/); }); test('an in-flight cascade member still refuses — it WILL be bound', () => { // Distinct from undeployed: a module mid-deploy is on its way to being // bound to the provider, so pausing around it would strand the transition. expect(() => planPause({ moduleId: 'greenwave', fleet: fleet({ caddy: 'DEPLOYING' }), cascade: true }), ).toThrow(/strand/); }); });