/** * What backup cadence a module actually has. * * One accessor, deliberately: the drift audit decides whether to ALERT * that a backup is stale, and the backup sweep decides whether to RUN * one. If those two read the manifest differently, a module can be * alerted-on-but-never-backed-up — an alert no human action can clear. * * The manifest is the author's SUGGESTION about a fleet they have never * seen; the operator's `backup_schedule` override wins. Resolution * happens HERE, at read time, so a corrected manifest still reaches * every install that has not overridden it. * * Absent from both means `daily`, not `manual`. Treating "nobody said" * as "never check and never run" is what let celilo-mgmt go 55 days * without a backup and forgejo and signal go without one entirely, all * silently. Opting out is a decision worth writing down, so it takes an * explicit `manual`. */ import type { ModuleManifest } from '../manifest/schema'; import { type Cadence, cadenceMs, parseCadence } from './cadence'; /** The `module_configs` key an operator's backup cadence is stored under. */ export const BACKUP_SCHEDULE_CONFIG_KEY = 'backup_schedule'; /** Used when neither the operator nor the manifest says anything. */ export const DEFAULT_BACKUP_SCHEDULE: Cadence = { minutes: 24 * 60 }; /** * `override` is the raw stored value of `backup_schedule`, or undefined when * the operator has set none. An unparseable override falls back to the * manifest rather than to `manual`: values are validated at SET time, so a bad * one here means hand-edited state, and the safe direction is backing up more * often than asked, never less. */ export function effectiveBackupSchedule( manifest: ModuleManifest, override: string | undefined, ): Cadence { if (override !== undefined) { const chosen = parseCadence(override); if (chosen !== null) return chosen; } const declared = manifest.backup?.schedule; if (declared !== undefined) { const suggested = parseCadence(declared); if (suggested !== null) return suggested; } return DEFAULT_BACKUP_SCHEDULE; } /** * Failures in a row after which retrying stops being worth the resources. * * Under this many, a failure is assumed transient and the module stays due on * the next tick, because most failures ARE transient — a storage endpoint * having a bad minute should not cost a full cadence period of coverage. * * At or over it, the evidence says otherwise and the retry slows to the * module's own cadence. Three is deliberately small: the useful information * from a retry is almost entirely in the first one or two, and the cost of * being wrong in this direction is bounded (one delayed backup) while the cost * of the other direction is not. */ export const MAX_RAPID_RETRIES = 3; /** What the backup history says about one module, for the due-ness decision. */ export interface BackupHistory { /** * When the last successful backup COMPLETED, or null if there has never * been one. Completion, not start — the same instant the freshness audit * measures from, so the run path and the alert path cannot disagree about * how old a backup is (design.md D6). */ lastSuccessAt: Date | null; /** Last attempt of any outcome, or null if none has ever been made. */ lastAttemptAt: Date | null; /** Attempts since the last success. */ consecutiveFailures: number; } /** * Whether to start a backup for a module right now. * * Due-ness used to be "has it been an interval since the last SUCCESS", which * is correct in the healthy case and degenerate in the failing one: a module * that cannot back up never advances that timestamp, so it is due at every * tick forever. The sweep runs hourly, so a *daily* module that started failing * was re-picked 24 times a day. On celilo-mgr that meant 20+ consecutive * forgejo attempts, each one assembling ~1.9 GB of staging and then dying, none * of them ever going to succeed for a reason no retry could change (celilo#685). * * Retrying is still right — it just cannot be unconditional. So the failure * count decides which clock applies: under `MAX_RAPID_RETRIES` the module stays * due against its last success, and at or over it the interval is measured from * the last ATTEMPT instead, which turns 24 doomed attempts a day into one. * * Backing off is not the same as going quiet. The `backups` drift monitor * measures staleness from the last success and is unaffected by this, so a * module that has slowed to one attempt a day still alerts as stale on exactly * the schedule it would have before — see services/audit/backups.ts. * * Pure, and time is a parameter, so the policy tests without a database. */ export function isBackupDueFromHistory( schedule: Cadence, history: BackupHistory, now: number, ): boolean { if (schedule === 'manual') return false; const interval = cadenceMs(schedule); if (history.consecutiveFailures >= MAX_RAPID_RETRIES) { // A run of failures with no attempt recorded is not a state the sweep can // produce, but "cannot prove an interval has passed" must not mean "start // another 1.9 GB attempt". if (!history.lastAttemptAt) return false; return now - history.lastAttemptAt.getTime() >= interval; } if (!history.lastSuccessAt) return true; return now - history.lastSuccessAt.getTime() >= interval; }