/** * `celilo backup sweep` — one pass of the scheduled backup runner. * * Invoked by the event-bus dispatcher on `timer.tick.1h`, not by a human. It is * a normal command so the dispatcher's existing subprocess isolation, retry, * and timeout apply unchanged. * * Thin adapter (Rule 10.5): compose the real dependencies, call runBackupSweep, * report counts. */ import { readdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { createModuleBackup, findBackupEligibleModules, isBackupDue, } from '../../services/backup-create'; import { failBackup, getBackup, listInProgressBackups } from '../../services/backup-metadata'; import { effectiveBackupRetention, pruneBackupsForModule, prunesNothing, } from '../../services/backup-retention'; import { BACKUP_SCHEDULE_CONFIG_KEY } from '../../services/backup-schedule'; import { STAGING_PREFIX, reapOrphanedStaging, resolveAbandonedBackups, } from '../../services/backup-staging'; import { type BackupSweepReport, runBackupSweep } from '../../services/backup-sweep'; import { configOverride } from '../../services/module-config'; import { isPidRunnable } from '../../services/module-operations'; import type { CommandResult } from '../types'; /** * Reclaim orphaned staging, then correct the records that were still claiming * to be running. * * The record fix-up lives here rather than inside the reaper because the reaper * is the filesystem decision and this is a database write; both follow from the * one liveness lookup, so neither repeats it. */ function reapStaging() { const report = reapOrphanedStaging({ listStagingDirs: () => { // A missing or unreadable temp dir is not a reason to fail the sweep. try { return readdirSync(tmpdir()) .filter((name) => name.startsWith(STAGING_PREFIX)) .map((name) => join(tmpdir(), name)); } catch { return []; } }, lookupOwner: (recordId) => { const record = getBackup(recordId); if (!record) return null; return { status: record.status, pid: record.pid, startedAt: record.startedAt }; }, isPidRunnable, remove: (path) => rmSync(path, { recursive: true, force: true }), now: () => Date.now(), }); return report; } /** * Correct every record that still claims to be running, staging or no staging. * * This replaces deriving the fix-up from what the reaper reclaimed. That * version only ever saw records whose directory still existed, so anything * cleared by a reboot or by hand stayed `in_progress` forever (#616). */ function resolveAbandonedRecords() { return resolveAbandonedBackups({ listInProgress: () => listInProgressBackups().map((b) => ({ id: b.id, pid: b.pid, startedAt: b.startedAt })), isPidRunnable, fail: (id, message) => failBackup(id, message), now: () => Date.now(), }); } export async function handleBackupSweep(): Promise { // Read once, so the cadence the sweep schedules on and the retention it // prunes with come from the same snapshot of the operator's config. const eligible = findBackupEligibleModules(); const configsByModule = new Map(eligible.map(({ module, configs }) => [module.id, configs])); const report = await runBackupSweep({ reapStaging, resolveAbandonedRecords, listEligible: () => eligible.map(({ module, manifest, configs }) => ({ id: module.id, manifest, scheduleOverride: configOverride(configs, BACKUP_SCHEDULE_CONFIG_KEY), })), isDue: (moduleId, schedule) => isBackupDue(moduleId, schedule), backup: (moduleId) => createModuleBackup(moduleId), prune: async ({ id, manifest }) => { const policy = effectiveBackupRetention(manifest, configsByModule.get(id)); if (prunesNothing(policy)) return; await pruneBackupsForModule(id, policy); }, }); const summary = formatReport(report); if (report.failures.length > 0) { return { success: false, error: summary }; } return { success: true, message: summary }; } /** * A quiet sweep stays quiet: counts that are zero are omitted. But a failure or * a lock-skip is always named, never aggregated away — "0 backed up" and * nothing else is indistinguishable from "nothing was due", which is exactly * how a backup runner that has silently stopped working looks. */ function formatReport(report: BackupSweepReport): string { const parts = [`${report.backedUp.length} backed up`]; if (report.skippedNotDue.length > 0) parts.push(`${report.skippedNotDue.length} not due`); if (report.skippedManual.length > 0) parts.push(`${report.skippedManual.length} manual`); if (report.skippedLocked.length > 0) parts.push(`${report.skippedLocked.length} locked`); if (report.failures.length > 0) parts.push(`${report.failures.length} FAILED`); const lines = [`backup sweep: ${parts.join(', ')}`]; // Reclamation is reported even though it is housekeeping: it is the only // visible evidence that backups have been dying, and a silent reaper would // hide the very failure it exists to clean up after. if (report.staging.reclaimed.length > 0) { lines.push( ` reclaimed ${report.staging.reclaimed.length} orphaned staging dir(s) from backups that were killed before cleanup`, ); for (const reclaimed of report.staging.reclaimed) { lines.push(` ${reclaimed.recordId} (${reclaimed.reason})`); } } if (report.records.resolved.length > 0) { lines.push( ` corrected ${report.records.resolved.length} record(s) that still claimed to be running`, ); } for (const moduleId of report.skippedLocked) { lines.push(` skipped ${moduleId}: another operation is in flight — retrying next tick`); } for (const failure of report.failures) { lines.push(` FAILED ${failure.moduleId}: ${failure.error}`); } return lines.join('\n'); }