import { describe, expect, test } from 'bun:test'; import type { ModuleManifest } from '../manifest/schema'; import { BACKUP_SWEEP_MAX_ATTEMPTS, BACKUP_SWEEP_PATTERN, BACKUP_SWEEP_SUBSCRIBER, BACKUP_SWEEP_TIMEOUT_MS, type BackupSweepDeps, type BackupSweepModule, ensureBackupSweepSubscriber, runBackupSweep, } from './backup-sweep'; import { InFlightError } from './module-operations'; function moduleWith(id: string, schedule?: string, scheduleOverride?: string): BackupSweepModule { const manifest = { id, hooks: { on_backup: { script: 'backup.ts' } }, ...(schedule ? { backup: { schedule } } : {}), } as unknown as ModuleManifest; return { id, manifest, scheduleOverride }; } function deps( modules: BackupSweepModule[], overrides: Partial = {}, ): BackupSweepDeps & { pruned: string[] } { const pruned: string[] = []; return { pruned, listEligible: () => modules, isDue: () => true, backup: async () => ({ success: true }), prune: async ({ id }) => { pruned.push(id); }, reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }), resolveAbandonedRecords: () => ({ resolved: [], kept: [] }), ...overrides, }; } describe('runBackupSweep', () => { // Ordering is the point, not just that it happens: this pass is the only // thing creating staging on a schedule, and a box already short on disk // needs the space back BEFORE another few GB are requested. test('reclaims orphaned staging before backing anything up', async () => { const order: string[] = []; const report = await runBackupSweep( deps([moduleWith('forgejo', 'daily')], { reapStaging: () => { order.push('reap'); return { reclaimed: [{ path: '/tmp/celilo-backup-x', recordId: 'x', reason: 'process-dead' }], kept: [], ignored: [], }; }, backup: async () => { order.push('backup'); return { success: true }; }, }), ); expect(order).toEqual(['reap', 'backup']); expect(report.staging.reclaimed).toHaveLength(1); }); // The sweep must resolve records whether or not the reaper found anything — // that independence IS the fix for #616. test('corrects abandoned records even when no staging was reclaimed', async () => { const report = await runBackupSweep( deps([moduleWith('forgejo', 'daily')], { reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }), resolveAbandonedRecords: () => ({ resolved: ['rec-a', 'rec-b'], kept: [] }), }), ); expect(report.staging.reclaimed).toEqual([]); expect(report.records.resolved).toEqual(['rec-a', 'rec-b']); }); test('reclaims staging even when no module is due to back up', async () => { const report = await runBackupSweep( deps([moduleWith('forgejo', 'daily')], { isDue: () => false, reapStaging: () => ({ reclaimed: [{ path: '/tmp/celilo-backup-y', recordId: 'y', reason: 'record-absent' }], kept: [], ignored: [], }), }), ); expect(report.backedUp).toEqual([]); expect(report.staging.reclaimed).toHaveLength(1); }); test('backs up a module whose declared cadence is due', async () => { const d = deps([moduleWith('authentik', 'daily')]); const report = await runBackupSweep(d); expect(report.backedUp).toEqual(['authentik']); expect(report.failures).toEqual([]); expect(d.pruned).toEqual(['authentik']); }); test('skips a module that is not yet due', async () => { const report = await runBackupSweep( deps([moduleWith('authentik', 'daily')], { isDue: () => false }), ); expect(report.backedUp).toEqual([]); expect(report.skippedNotDue).toEqual(['authentik']); }); test('never auto-backs-up an explicit manual schedule', async () => { const report = await runBackupSweep(deps([moduleWith('scratch', 'manual')])); expect(report.backedUp).toEqual([]); expect(report.skippedManual).toEqual(['scratch']); }); test("an operator's override decides the cadence, not the manifest", async () => { const d = deps([moduleWith('caddy', 'daily', '6h')]); const seen: Array<[string, unknown]> = []; d.isDue = (moduleId, schedule) => { seen.push([moduleId, schedule]); return true; }; await runBackupSweep(d); expect(seen).toEqual([['caddy', { minutes: 360 }]]); }); test('an override of manual stops a module the manifest wanted backed up', async () => { const report = await runBackupSweep(deps([moduleWith('caddy', 'daily', 'manual')])); expect(report.backedUp).toEqual([]); expect(report.skippedManual).toEqual(['caddy']); }); test('an undeclared schedule is backed up, not treated as manual', async () => { // The regression this whole subsystem exists for: forgejo and signal // declare no `backup:` block and had never been backed up. const report = await runBackupSweep(deps([moduleWith('forgejo')])); expect(report.backedUp).toEqual(['forgejo']); expect(report.skippedManual).toEqual([]); }); test('a held operation lock is a skip, not a failure, and stops the pass', async () => { const d = deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], { backup: async () => { throw new InFlightError([]); }, }); const report = await runBackupSweep(d); expect(report.skippedLocked).toEqual(['authentik']); expect(report.failures).toEqual([]); expect(report.backedUp).toEqual([]); expect(d.pruned).toEqual([]); }); test('a failed backup is recorded and the remaining modules still run', async () => { const report = await runBackupSweep( deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], { backup: async (moduleId) => moduleId === 'authentik' ? { success: false, error: 'hook exited 1' } : { success: true }, }), ); expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'hook exited 1' }]); expect(report.backedUp).toEqual(['forgejo']); }); test('a thrown storage error fails only that module', async () => { const report = await runBackupSweep( deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], { backup: async (moduleId) => { if (moduleId === 'authentik') throw new Error('Storage not verified'); return { success: true }; }, }), ); expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'Storage not verified' }]); expect(report.backedUp).toEqual(['forgejo']); }); test('does not prune when a backup failed', async () => { const d = deps([moduleWith('authentik', 'daily')], { backup: async () => ({ success: false, error: 'nope' }), }); await runBackupSweep(d); expect(d.pruned).toEqual([]); }); }); describe('ensureBackupSweepSubscriber', () => { test('registers the hourly sweep handler', () => { const calls: Array<{ name: string; pattern: string; handler: string; registeredBy?: string; maxAttempts?: number; timeoutMs?: number; }> = []; ensureBackupSweepSubscriber({ subscribe: (options) => calls.push(options), }); expect(calls).toEqual([ { name: BACKUP_SWEEP_SUBSCRIBER, pattern: BACKUP_SWEEP_PATTERN, handler: 'celilo backup sweep', registeredBy: 'celilo-backup', maxAttempts: BACKUP_SWEEP_MAX_ATTEMPTS, timeoutMs: BACKUP_SWEEP_TIMEOUT_MS, }, ]); // 1h is the coarsest tick that can still serve an `hourly` cadence. expect(BACKUP_SWEEP_PATTERN).toBe('timer.tick.1h'); }); // Both values are the bug. Registering without them inherits the bus // defaults of 60000ms / 3 attempts, which cannot finish a backup that needs // ~5.5 minutes and then retries the impossible twice more per tick — three // killed backups and three stranded staging directories every hour. test('states the budget explicitly rather than inheriting the bus defaults', () => { let registered: { maxAttempts?: number; timeoutMs?: number } | undefined; ensureBackupSweepSubscriber({ subscribe: (options) => { registered = options; return options; }, }); expect(registered?.timeoutMs).toBeDefined(); expect(registered?.maxAttempts).toBeDefined(); expect(registered?.timeoutMs).toBeGreaterThan(60_000); expect(registered?.maxAttempts).toBe(1); }); });