/** * The thing that actually runs backups on a schedule. * * A manifest's `backup.schedule` was decorative until this existed: the drift * audit reported staleness and `celilo module backup` needed a human. This is * the pass that makes a module declaring `daily` get backed up daily with * nobody watching. * * No new scheduling mechanism. Registered as an ordinary bus subscriber whose * handler is `celilo backup sweep` against `timer.tick.1h`, exactly like the * alerting sweep (`services/alerting/monitors.ts`) — the timer already exists, * already survives restarts, and already has retry and dedup. One hour is the * coarsest tick that can still serve an `hourly` cadence; `1d` cannot. * * Dependencies are injected so the pass itself tests with no database, no * storage, and no hook execution. */ import type { ModuleManifest } from '../manifest/schema'; import { effectiveBackupSchedule } from './backup-schedule'; import type { ReapStagingReport, ResolveAbandonedReport } from './backup-staging'; import { BACKUP_SWEEP_PATTERN, type Cadence } from './cadence'; import { InFlightError } from './module-operations'; export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep'; // The tick itself lives in services/cadence.ts, next to the floor derived from // it — a sweep whose tick and whose finest servable cadence are stated in two // files is the pair that drifts. export { BACKUP_SWEEP_PATTERN }; /** * How long the sweep may run before the dispatcher kills it. * * Set EXPLICITLY because the bus default is 60 seconds and this pass cannot * finish in 60 seconds. A single forgejo backup measured 2026-08-06 takes ~5.5 * minutes — snapshotting SQLite, archiving repositories, and streaming ~1.3 GB * over SSH — and the sweep runs every due module serially, so the budget covers * the sum rather than the slowest one. * * Inheriting the default made every scheduled forgejo backup structurally * impossible: killed at 60s, mid-encrypt, three times an hour, for days. * * Four hours is generous on purpose. The cost of it being too large is one * delayed reclamation of a wedged sweep; the cost of it being too small is a * backup that can never succeed. `celilo-mgmt.registry-poll` set the precedent * at 30 minutes for the same reason. * * ponytail: one flat number for a serial sweep. If the fleet grows enough that * the sum stops fitting, the upgrade is for the sweep to emit a per-module * backup event carrying its own budget. */ export const BACKUP_SWEEP_TIMEOUT_MS = 4 * 60 * 60 * 1000; /** * One attempt per tick. * * The bus default of 3 is right for a transient fault and wrong for this pass. * A sweep that cannot finish inside its budget will not finish on the retry * either — retrying it produced three killed backups and three stranded staging * directories per hour instead of one, which is how 27 GB accumulated in 5.7 * hours. The hourly tick is already the retry. */ export const BACKUP_SWEEP_MAX_ATTEMPTS = 1; export interface SubscriberRegistrar { subscribe(options: { name: string; pattern: string; handler: string; registeredBy?: string; maxAttempts?: number; timeoutMs?: number; }): unknown; } /** * Idempotent: `bus.subscribe` upserts by name, so this is safe to call on * every module install and update. * * Also called from `celilo system migrate`, which the .deb postinst runs on * every apt upgrade. Registering only from module install/update meant a * corrected budget would not reach an existing fleet until some module happened * to be touched next — indistinguishable from a fix that shipped and silently * did nothing. celilo-mgr sat at the 60s default with the row already present. * * This re-arms a deliberately paused sweep, and that is intentional: the pause * only ever existed because staging leaked, and the reaper that makes it not * leak ships in this same binary. Re-arming without the reaper present is the * failure this comment exists to prevent — do not lift this call into a release * that does not carry `reapOrphanedStaging`. */ export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void { bus.subscribe({ name: BACKUP_SWEEP_SUBSCRIBER, pattern: BACKUP_SWEEP_PATTERN, handler: 'celilo backup sweep', registeredBy: 'celilo-backup', maxAttempts: BACKUP_SWEEP_MAX_ATTEMPTS, timeoutMs: BACKUP_SWEEP_TIMEOUT_MS, }); } export interface BackupSweepModule { id: string; manifest: ModuleManifest; /** * The operator's `backup_schedule` override, or undefined when they have set * none. Carried rather than pre-resolved so the cadence is resolved through * the one shared accessor here, the same way the freshness audit resolves it. */ scheduleOverride: string | undefined; } export interface BackupSweepDeps { /** Installed modules that declare an `on_backup` hook. */ listEligible(): BackupSweepModule[]; isDue(moduleId: string, schedule: Cadence): boolean; backup(moduleId: string): Promise<{ success: boolean; error?: string }>; /** Apply the module's declared retention. No-op when it declares none. */ prune(module: BackupSweepModule): Promise; /** Reclaim staging left by backups whose process died. See backup-staging.ts. */ reapStaging(): ReapStagingReport; /** * Correct records that still claim to be running after their process died. * * Separate from `reapStaging` on purpose. Deriving this from what the reaper * happened to reclaim left records stranded forever once their staging was * gone — #616. */ resolveAbandonedRecords(): ResolveAbandonedReport; } export interface BackupSweepReport { /** Staging reclaimed before this pass created any of its own. */ staging: ReapStagingReport; /** Records corrected from a stale `in_progress`. */ records: ResolveAbandonedReport; backedUp: string[]; /** Effective cadence `manual` — the operator or the author opted out. */ skippedManual: string[]; skippedNotDue: string[]; /** Another module operation held the lock. Not a failure; retried next tick. */ skippedLocked: string[]; failures: Array<{ moduleId: string; error: string }>; } export async function runBackupSweep(deps: BackupSweepDeps): Promise { // Reclaim BEFORE backing anything up, not after. This pass is the only thing // that creates staging on a schedule, so it is where the orphans come from — // and a box already short on disk needs the space freed before we ask for // another ~4 GB of it, not once we are finished with it. const staging = deps.reapStaging(); // Independent of the reap above, and that independence is the fix for #616: // a record whose staging is already gone is invisible to the reaper, so // resolving records off the reaper's results left 107 of them stranded. const records = deps.resolveAbandonedRecords(); const report: BackupSweepReport = { staging, records, backedUp: [], skippedManual: [], skippedNotDue: [], skippedLocked: [], failures: [], }; for (const module of deps.listEligible()) { const schedule = effectiveBackupSchedule(module.manifest, module.scheduleOverride); if (schedule === 'manual') { report.skippedManual.push(module.id); continue; } if (!deps.isDue(module.id, schedule)) { report.skippedNotDue.push(module.id); continue; } let result: { success: boolean; error?: string }; try { result = await deps.backup(module.id); } catch (error) { // Refusing to run while a deploy/restore is in flight is correct, and a // scheduled run must not bypass it. The lock is global, so once it is // held every remaining module would refuse identically — stop the pass // rather than collect the same refusal N times. The next tick retries. if (error instanceof InFlightError) { report.skippedLocked.push(module.id); break; } // Everything else — an unverified storage destination, a missing default // — throws before a backup row exists. Record it against the module and // keep going; one module's failure must not cancel the rest. report.failures.push({ moduleId: module.id, error: error instanceof Error ? error.message : String(error), }); continue; } if (!result.success) { report.failures.push({ moduleId: module.id, error: result.error ?? 'backup failed' }); continue; } report.backedUp.push(module.id); await deps.prune(module); } return report; }