/** * Disk-usage probe for every system celilo can reach. * * Structurally a sibling of `machine-probe.ts` — same SSH bounding — with two * deliberate differences, and they are the whole reason this file has its own * comment. * * ⚠️ IT PROBES SYSTEMS, NOT JUST MACHINES. * * `machine` has celilo's narrow meaning: an operator-pre-provisioned box in the * machine pool. It excludes every LXC and VM celilo provisioned itself through a * container_service — which is most of the fleet, and all of the interesting * parts of it. This probe walked `listMachines()` alone, so the registry, the * forge and the firewall were never measured at all. * * That is not a coverage nicety. Two instances filled on the same night, and * neither raised anything (celilo#1133): * * vmid 204, celilo-registry — 20G, 100%, four kilobytes free. Every celilo * surface stayed green, because reads work on a full disk and only writes * fail. The health check passed, the sparse index served, `module search` * returned all 42 modules. The first symptom was a release dying with * `Internal Server Error`, and finding out why took an ssh and a `df`. * vmid 206, git.celilo.computer — 40G, 100%. The builder's runner logged * `failed to fetch task … database or disk is full` every two seconds and * claimed no work for ~13 hours. It presented as a DEAD CI RUNNER. The * runner was healthy, registered and polling the whole time. * * Both are container_service instances, so both were invisible here for the same * reason, and each wore a different disguise. Scheduling the `disk_space` monitor * would not have caught either: the check would have gone on reporting all-clear * about filesystems it never looked at, which is worse than not having it. * * So the unit here is the SYSTEM: the machine pool plus every provisioned * instance with an address. * * ⚠️ THE LOCAL BOX IS MEASURED, NOT EXEMPTED. * * `probeMachines()` reports the management server reachable WITHOUT probing it, * and that is correct there: celilo runs as a user with no SSH key for itself, * so `ssh root@127.0.0.1` fails on a perfectly healthy host — and the question * is meaningless anyway, since if this box were unreachable this code would not * be running. * * None of that transfers to disk. The management server stages backups, caches * modules, holds the celilo DB and writes the logs; it is the likeliest host in * the fleet to fill, and it is the host that DID fill. A disk check that copied * the probe's structure and inherited its local shortcut would skip the only * machine the check exists to protect — reporting "all clear" about a * filesystem it never looked at. * * So the local box reads `statfs` directly. There is no reachability question * to answer about the machine running the code, only a usage one. */ import { execFile } from 'node:child_process'; import { statfs } from 'node:fs/promises'; import { promisify } from 'node:util'; import type { DbClient } from '../db/client'; import type { DiskUsageResult } from './audit/disk-space'; import { getProvisionedSystems } from './deployed-systems'; import { listMachines } from './machine-pool'; import { LOCAL_MACHINE_IP } from './ssh-key-manager'; const execFileAsync = promisify(execFile); /** * Percent-used the way `df` reports it. * * Deliberately not `1 - bavail/blocks`. A filesystem reserves blocks for root, * so free-to-root and free-to-everyone-else differ; `df` computes capacity * against the space an ordinary process can actually use, and an operator * comparing this alert to their own `df` output must see the same number. */ export function percentUsed(totalBlocks: number, freeBlocks: number, availBlocks: number): number { const used = totalBlocks - freeBlocks; const usable = used + availBlocks; if (usable <= 0) return 0; return Math.round((used / usable) * 100); } /** Parse the data row of `df -P /`. Returns null when the output is unusable. */ export function parseDfOutput( stdout: string, ): { usedPercent: number; availableBytes: number } | null { // -P guarantees one line per filesystem, so the row we want is the second. const line = stdout.trim().split('\n')[1]; if (!line) return null; // Filesystem 1024-blocks Used Available Capacity Mounted-on const fields = line.trim().split(/\s+/); if (fields.length < 5) return null; const available = Number(fields[3]); const percent = Number(fields[4]?.replace('%', '')); if (!Number.isFinite(available) || !Number.isFinite(percent)) return null; return { usedPercent: percent, availableBytes: available * 1024 }; } async function probeLocal(hostname: string, ipAddress: string): Promise { try { const stats = await statfs('/'); return { hostname, ipAddress, usedPercent: percentUsed(stats.blocks, stats.bfree, stats.bavail), availableBytes: stats.bavail * stats.bsize, }; } catch (err) { return { hostname, ipAddress, usedPercent: null, message: err instanceof Error ? err.message : String(err), }; } } async function probeRemote( hostname: string, ipAddress: string, sshUser: string, ): Promise { try { // Same bounding as machine-probe: BatchMode so a password prompt can never // hang the probe, ConnectTimeout so an unresponsive host — the condition // this check exists to notice — cannot wedge it. const { stdout } = await execFileAsync( 'ssh', [ '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=5', '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', `${sshUser}@${ipAddress}`, 'df -P /', ], { timeout: 8000 }, ); const parsed = parseDfOutput(stdout); if (!parsed) { return { hostname, ipAddress, usedPercent: null, message: `unparseable df output: ${stdout}`, }; } return { hostname, ipAddress, ...parsed }; } catch (err) { const e = err as { stderr?: string; message?: string }; return { hostname, ipAddress, usedPercent: null, message: e.stderr?.trim() || e.message || 'df probe failed', }; } } /** * Every system worth measuring, deduplicated by address. * * A module with no `requires.system` (an API-only module like namecheap) has no * instance and contributes nothing. Two modules co-hosted on one container * contribute one entry, not two — the filesystem is shared and so is its * finding. * * celilo provisions containers with its own key and deploys to them as root * (the `ansible_user` default), which is the same access `df` needs here. */ export function diskProbeTargets( machines: Array<{ hostname: string; ipAddress: string; sshUser: string }>, systems: Array<{ hostname: string; ipv4Address: string; vmid: number | null }>, ): Array<{ hostname: string; ipAddress: string; sshUser: string }> { const byAddress = new Map(); for (const m of machines) byAddress.set(m.ipAddress, m); for (const s of systems) { // No vmid means no celilo-provisioned instance behind this row (a // machine-pool placement), and the machine pool above already covers it. if (s.vmid == null || !s.ipv4Address) continue; if (byAddress.has(s.ipv4Address)) continue; byAddress.set(s.ipv4Address, { hostname: s.hostname, ipAddress: s.ipv4Address, sshUser: 'root', }); } return [...byAddress.values()]; } export async function probeDiskUsage(db: DbClient): Promise { const targets = diskProbeTargets(await listMachines(), getProvisionedSystems(db)); return Promise.all( targets.map((t) => t.ipAddress === LOCAL_MACHINE_IP ? probeLocal(t.hostname, t.ipAddress) : probeRemote(t.hostname, t.ipAddress, t.sshUser), ), ); }