/** * `celilo module operations` — see and release the module-operation lock. * * Backup and restore refuse to run while another operation is in flight. * When that refusal is wrong, the operator previously had no way to see * the lock at all, let alone clear it: the error said "wait for it to * complete", which for a suspended process is advice that can never come * true. A `module deploy` Ctrl-Z'd on a lost terminal blocked every * backup on the fleet for 20 days on exactly that advice. * * `clear` marks rows failed rather than deleting them — the history of * what was abandoned, and when, is worth more than a tidy table. * * That is load-bearing now, not merely tidy: `clear` runs hourly off the * bus, and `services/audit/abandoned-operations.ts` reads exactly these * released rows to notice that one module's backup is being killed over * and over. Turning this into a DELETE would tidy the table and silently * destroy the only signal that a repeatedly-dying operation ever leaves. */ import { and, eq } from 'drizzle-orm'; import { getDb } from '../../db/client'; import { type ModuleOperation, moduleOperations } from '../../db/schema'; import { ABANDONED_RELEASE_MESSAGE, OPERATION_TTL_MS, isPidRunnable, } from '../../services/module-operations'; import type { CommandResult } from '../types'; /** Why a row is not holding the lock, or null when it still is. */ function abandonedReason(row: ModuleOperation, now: number): string | null { if (now - row.startedAt.getTime() > OPERATION_TTL_MS) return 'expired'; if (!isPidRunnable(row.pid)) return 'not running'; return null; } function formatAge(ms: number): string { const minutes = Math.floor(ms / 60_000); if (minutes < 60) return `${minutes}m`; const hours = Math.floor(minutes / 60); if (hours < 24) return `${hours}h`; return `${Math.floor(hours / 24)}d`; } function inProgressRows(): ModuleOperation[] { return getDb() .select() .from(moduleOperations) .where(eq(moduleOperations.status, 'in_progress')) .all(); } /** * Abandoned rows are summarised, not listed, unless `--abandoned` asks for * them. The question this command answers is "what holds the lock right * now", and on a fleet where something is dying repeatedly the answer was * buried under 85 corpses. The count still prints, so they never become * invisible — the audit is what reads them as a symptom * (`services/audit/abandoned-operations.ts`). */ function handleList(flags: Record): CommandResult { const now = Date.now(); const rows = inProgressRows(); const showAbandoned = flags.abandoned === true || flags.all === true; if (rows.length === 0) { console.log('\nNo module operations in progress.\n'); return { success: true, message: 'no operations in progress' }; } const holding = rows.filter((row) => abandonedReason(row, now) === null); const abandoned = rows.length - holding.length; const shown = showAbandoned ? rows : holding; console.log('\nModule operations in progress:\n'); for (const row of shown) { const reason = abandonedReason(row, now); const age = formatAge(now - row.startedAt.getTime()); const status = reason ? `abandoned (${reason})` : 'HOLDING LOCK'; console.log( ` ${row.operation.padEnd(9)} ${row.moduleId.padEnd(16)} pid ${String(row.pid).padEnd(8)} ${age.padStart(4)} ago ${status}`, ); } if (shown.length === 0) console.log(' (nothing is holding the lock)'); console.log(''); if (abandoned > 0 && !showAbandoned) { console.log( `${abandoned} abandoned row(s) hidden — "--abandoned" lists them, the hourly sweep clears them.\n`, ); } return { success: true, message: `${rows.length} in progress (${holding.length} holding the lock, ${abandoned} abandoned)`, }; } /** * Release abandoned rows. `--all` also releases rows whose process still * looks alive. * * The permissive form exists because the pathological case is precisely * the one where our liveness detection was wrong — a pid that has been * recycled by an unrelated process reads as perfectly healthy. Refusing * to clear it would recreate the outage this command exists to end. It * is opt-in and names what it is overriding. */ function handleClear(flags: Record): CommandResult { const db = getDb(); const now = Date.now(); const force = flags.all === true; const rows = inProgressRows(); const targets = force ? rows : rows.filter((row) => abandonedReason(row, now) !== null); if (targets.length === 0) { const held = rows.length; if (held > 0) { return { success: true, message: `Nothing to clear — ${held} operation(s) still look genuinely in flight. Use --all to release them anyway.`, }; } return { success: true, message: 'Nothing to clear — no operations in progress.' }; } for (const row of targets) { db.update(moduleOperations) .set({ status: 'failed', completedAt: new Date(), errorMessage: ABANDONED_RELEASE_MESSAGE, }) .where(and(eq(moduleOperations.id, row.id), eq(moduleOperations.status, 'in_progress'))) .run(); console.log(` released ${row.operation} of ${row.moduleId} (pid ${row.pid})`); } return { success: true, message: `Released ${targets.length} operation(s)` }; } export function handleModuleOperations( args: string[], flags: Record, ): CommandResult { const action = args[0]; if (!action || action === 'list') return handleList(flags); if (action === 'clear') return handleClear(flags); return { success: false, error: `Unknown action "${action}"\n\nUsage: celilo module operations [list|clear] [--abandoned] [--all]`, }; }