/** * Backup freshness drift check. * * For each installed module that declares an `on_backup` hook, decides whether * the most recent successful backup is too old, against the module's EFFECTIVE * cadence — the operator's override if they set one, else the manifest's * suggestion. The threshold is that cadence plus a grace allowance, so a * slightly-late scheduled run doesn't flag drift on every audit. * * Modules without an `on_backup` hook are skipped — there's nothing to back up. * A module whose effective cadence is `manual` is skipped entirely: it has * opted out, and a finding asking it to back up is one no operator action can * clear, because the action it asks for is the one they declined. An unset * cadence is `daily`, not `manual` — see [[services/backup-schedule.ts]] for * why that default matters. * * Time is injected so tests can pin "now" deterministically. */ import type { ModuleManifest } from '../../manifest/schema'; import { effectiveBackupSchedule } from '../backup-schedule'; import { type Cadence, cadenceMs, formatCadence } from '../cadence'; import type { DriftFinding } from './types'; export interface InstalledModuleBackupInfo { id: string; /** Lifecycle state — non-deployed modules have nothing to back up. */ state: string; manifest: ModuleManifest; /** The operator's `backup_schedule` override, or undefined when unset. */ scheduleOverride: string | undefined; /** Most recent successful backup timestamp (ms since epoch), or null. */ lastSuccessfulBackupAt: number | null; } export interface BackupsAuditDeps { modules: InstalledModuleBackupInfo[]; /** * Test-only override: forces this threshold for every module whose * effective cadence is not `manual`. Production code never sets this — * cadence-derived thresholds are the right behavior. */ staleAfterMs?: number; /** Defaults to `Date.now()`. */ now?: () => number; } const HOUR = 60 * 60 * 1000; const DAY = 24 * HOUR; /** * How old a backup may get before it is drift: the cadence itself plus a grace * allowance, so a legitimately-just-late run doesn't trip the audit. * * A formula rather than a table because a table cannot answer for `6h`, and an * operator who sets a custom cadence would get either no staleness reporting or * an arbitrary threshold. The four previous values were not a formula in * disguise — hourly had 100% grace, daily 4%, weekly 14%, monthly 7% — so this * moves them: daily 25h → 26.4h, weekly 8d → 7.7d (the only one that alerts * EARLIER), monthly 32d → 33d, hourly unchanged at 2h. See design.md D7. */ export function backupStaleThresholdMs(cadence: Cadence): number | null { if (cadence === 'manual') return null; const interval = cadenceMs(cadence); return interval + Math.max(HOUR, interval * 0.1); } /** * Whether celilo can back this module up at all. * * Exported because it is the gate `backup-create.ts` actually applies, and the * console has to draw the same distinction. On the live fleet five of 23 * deployed modules declare the hook: the other eighteen are not overdue and not * failing, there is simply nothing to run (celilo#1131). A second copy of this * test elsewhere would eventually disagree about which eighteen. */ export function moduleHasBackupHook(manifest: ModuleManifest): boolean { return Boolean(manifest.hooks?.on_backup); } function formatAge(ms: number): string { const days = Math.floor(ms / DAY); if (days >= 1) return `${days}d`; const hours = Math.floor(ms / HOUR); if (hours >= 1) return `${hours}h`; const mins = Math.floor(ms / (60 * 1000)); return `${mins}m`; } const DEPLOYED_STATES = new Set(['INSTALLED', 'VERIFIED']); export async function auditBackups(deps: BackupsAuditDeps): Promise { const now = (deps.now ?? Date.now)(); const findings: DriftFinding[] = []; for (const m of deps.modules) { if (!moduleHasBackupHook(m.manifest)) continue; // Non-deployed modules have no live state to back up. Surfacing // a "no successful backup recorded" finding for an IMPORTED // module is just noise — the operator hasn't deployed yet, so // there's nothing to lose. Skip them entirely from the backup // audit. if (!DEPLOYED_STATES.has(m.state)) continue; const cadence = effectiveBackupSchedule(m.manifest, m.scheduleOverride); // BEFORE the never-backed-up check, not after. A module opted out of // scheduled backups was reported as missing one forever, and the only // remediation offered was the very thing the operator declined. if (cadence === 'manual') continue; if (m.lastSuccessfulBackupAt === null) { findings.push({ category: 'backups', severity: 'drift', code: 'backup_missing', message: `${m.id}: no successful backup recorded (schedule: ${formatCadence(cadence)})`, remediation: `celilo backup create ${m.id} --force`, actionable: true, subject: m.id, }); continue; } const threshold = deps.staleAfterMs ?? backupStaleThresholdMs(cadence); if (threshold === null) continue; const age = now - m.lastSuccessfulBackupAt; if (age > threshold) { findings.push({ category: 'backups', severity: 'drift', code: 'backup_stale', message: `${m.id}: last successful backup is ${formatAge(age)} old (schedule: ${formatCadence(cadence)}, threshold: ${formatAge(threshold)})`, remediation: `celilo backup create ${m.id} --force`, actionable: true, subject: m.id, }); } } return findings; }