/** * Disk-space check. * * Catches the failure that had no detector at all: a filesystem filling up. * celilo-mgr reached 34% and climbing at ~4.8 GB/hour from leaked backup * staging, and the only reason anyone noticed was an operator running `df` by * eye. Every one of the eighteen configured monitors watched module health or * machine reachability; not one looked at disk. * * Reports EARLY, not at exhaustion. A check that fires once a filesystem is * full reports an outage instead of preventing one, so the thresholds leave * room to act: `drift` at 85% is "you have time", `blocked` at 95% is "you do * not". On a 117 GB root that is ~17 GB and ~6 GB of headroom respectively — * hours at the leak rate that motivated this, days at any normal one. * * The audit consumes pre-computed measurements so it stays unit-testable * without a live filesystem or SSH client, exactly like `machines-reachable`. */ import type { DriftFinding } from './types'; /** Usage at or above this is divergence worth an operator's attention. */ export const DISK_DRIFT_PERCENT = 85; /** Usage at or above this is close enough to exhaustion to gate on. */ export const DISK_BLOCKED_PERCENT = 95; export interface DiskUsageResult { /** * User-facing hostname, and the identifier the finding is keyed by. * * NOT the machine's UUID. Suppression resolves a machine's ancestor key from * its hostname (`machineAlertKey` in alerting/suppression.ts), so a finding * subjected on the UUID produces an alert key suppression can never match — * which is exactly the bug filed as #596 against `machines_reachable`. Using * the hostname here also satisfies CLAUDE.md: users never see UUIDs. */ hostname: string; ipAddress: string; /** Percent of the root filesystem in use, or null when it could not be measured. */ usedPercent: number | null; /** Bytes still available. Omitted when unmeasured. */ availableBytes?: number; /** Why the measurement failed, when `usedPercent` is null. */ message?: string; } export interface DiskSpaceAuditDeps { results: DiskUsageResult[]; } function humanBytes(bytes: number): string { const units = ['B', 'KB', 'MB', 'GB', 'TB']; let value = bytes; let unit = 0; while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit++; } return `${value.toFixed(value < 10 && unit > 0 ? 1 : 0)} ${units[unit]}`; } export function auditDiskSpace(deps: DiskSpaceAuditDeps): DriftFinding[] { const findings: DriftFinding[] = []; for (const result of deps.results) { // Unmeasurable is NOT healthy — but it does not page either. The host is // already unreachable, `machines_reachable` is already alerting on it, and // a second page for one dead host is noise. This used to be filed as `todo` // because that was the only non-paging severity available; `unmeasured` // (D7) is what it always meant, and unlike `todo` it stops the verdict // returning READY. if (result.usedPercent === null) { findings.push({ category: 'disk_space', severity: 'unmeasured', code: 'disk_unmeasured', message: `${result.hostname}: disk usage could not be measured`, details: result.message, remediation: 'The host is unreachable or `df` failed on it. `machines_reachable` covers reachability; this finding only records that disk is currently unknown, not that it is healthy.', actionable: false, subject: result.hostname, }); continue; } if (result.usedPercent < DISK_DRIFT_PERCENT) continue; const critical = result.usedPercent >= DISK_BLOCKED_PERCENT; const free = result.availableBytes === undefined ? '' : `, ${humanBytes(result.availableBytes)} free`; findings.push({ category: 'disk_space', severity: critical ? 'blocked' : 'drift', code: critical ? 'disk_critical' : 'disk_low', message: `${result.hostname}: root filesystem ${result.usedPercent}% full${free}`, details: critical ? `At or above ${DISK_BLOCKED_PERCENT}% the host is close enough to exhaustion that writes can begin failing. On the management server a full root filesystem takes the event bus and dispatcher with it, including whatever scheduled work would otherwise clean up.` : `At or above ${DISK_DRIFT_PERCENT}% there is still room to act. Find the growth before it becomes an outage rather than after.`, remediation: [ `Find what is growing on ${result.hostname}:`, ' df -h /', ' sudo du -xh --max-depth=2 / | sort -h | tail -30', 'Then fix the source — a retention policy, a size cap, or a prune', 'schedule. A one-off delete leaves the same thing growing.', ].join('\n'), // Multi-step diagnosis, not a one-shot celilo command. actionable: false, subject: result.hostname, }); } return findings; }