/** * Repeatedly-abandoned module operations — reading a pile of corpses as a * symptom. * * An operation whose process dies without recording an outcome leaves an * in_progress row behind. Those rows now get reclaimed hourly, which fixes * the clutter but would destroy the more valuable half of the signal: 60+ * abandoned `backup` rows for one module is not clutter, it is a backup * being killed before it can finish, and it went unnoticed for days * precisely because nobody reads a pile as a symptom. * * Reclaiming marks rows `failed` with `ABANDONED_RELEASE_MESSAGE` rather * than deleting them, so the sweep preserves exactly the history this * check counts. One abandonment is an operator who hit Ctrl-C; a handful * in a week is something killing the same operation over and over. * * Pure: the rows are injected, so the judgement tests with no database. */ import { and, eq } from 'drizzle-orm'; import type { DbClient } from '../../db/client'; import { type ModuleOperationKind, moduleOperations } from '../../db/schema'; import { ABANDONED_RELEASE_MESSAGE } from '../module-operations'; import type { DriftFinding } from './types'; /** One reclaimed row, flattened to what the judgement needs. */ export interface AbandonedOperationRecord { moduleId: string; operation: ModuleOperationKind; /** When the operation started, epoch ms. */ startedAt: number; } export interface AbandonedOperationsAuditDeps { records: AbandonedOperationRecord[]; /** Defaults to `Date.now()`. */ now?: () => number; } const DAY = 24 * 60 * 60 * 1000; /** * How far back to look, and how many abandonments in that window stop * being noise. * * Seven days spans a weekly backup cadence, so a module that only runs one * operation a week still needs a genuine repeat to trip. Three is the * smallest count that cannot be one operator interrupting one deploy. */ export const ABANDONED_WINDOW_MS = 7 * DAY; export const ABANDONED_THRESHOLD = 3; /** * Findings are per (module, operation kind): a module whose backups are * being killed and whose deploys are fine should say so, rather than * merging into an unactionable "12 operations abandoned". */ export function auditAbandonedOperations(deps: AbandonedOperationsAuditDeps): DriftFinding[] { const now = (deps.now ?? Date.now)(); const counts = new Map(); for (const record of deps.records) { if (now - record.startedAt > ABANDONED_WINDOW_MS) continue; const key = `${record.moduleId}${record.operation}`; const entry = counts.get(key) ?? { moduleId: record.moduleId, operation: record.operation, count: 0, }; entry.count += 1; counts.set(key, entry); } const findings: DriftFinding[] = []; for (const { moduleId, operation, count } of counts.values()) { if (count < ABANDONED_THRESHOLD) continue; findings.push({ category: 'abandoned_operations', severity: 'drift', code: 'operations_repeatedly_abandoned', subject: moduleId, message: `${moduleId}: ${count} ${operation} operations abandoned in the last 7d — the operation is being killed before it records an outcome`, details: 'An abandoned operation means the process died without writing success or failure — a timeout, a SIGTERM, or a crash. Repeated abandonment of the same operation is a broken operation, not stale bookkeeping.', remediation: `celilo module journal ${moduleId}`, actionable: true, }); } return findings.sort((a, b) => a.subject.localeCompare(b.subject)); } /** * The rows the check runs against: operations the sweep reclaimed. * * Matching on the release message rather than on `status = 'failed'` alone * is the whole distinction — a failed operation reported its failure and is * already visible; an abandoned one reported nothing. * * Shared by `celilo system audit` and the schedulable `abandoned_operations` * monitor, so both judge the fleet from the same rows. */ export function loadAbandonedOperations(db: DbClient): AbandonedOperationRecord[] { return db .select() .from(moduleOperations) .where( and( eq(moduleOperations.status, 'failed'), eq(moduleOperations.errorMessage, ABANDONED_RELEASE_MESSAGE), ), ) .all() .map((row) => ({ moduleId: row.moduleId, operation: row.operation, startedAt: row.startedAt.getTime(), })); }