import { describe, expect, test } from 'bun:test'; import { diskProbeTargets, parseDfOutput, percentUsed } from './disk-probe'; describe('percentUsed', () => { // Checked against what celilo-mgr's df actually printed, because an operator // comparing an alert to their own df must see the same figure. // // 117G root, 11G used, 102G available — after the staging was cleared. test('matches df for celilo-mgr at 10% used', () => { expect(percentUsed(122_683_392, 111_149_056, 106_954_752)).toBe(10); }); // Same filesystem at 38G used / 75G available — what df reported while the // leak was running. test('matches df for celilo-mgr at 34% used', () => { expect(percentUsed(122_683_392, 82_837_504, 78_643_200)).toBe(34); }); // Not 1 - bavail/blocks. A filesystem reserves blocks for root, so // free-to-root and free-to-everyone differ; df computes capacity against // what an ordinary process can use. With a large reserve the naive formula // over-reports and would page early forever. test('excludes root-reserved blocks, as df does', () => { // 1000 total, 100 free to root, only 50 usable by others → 900 used of 950. expect(percentUsed(1000, 100, 50)).toBe(95); // The naive 1 - bavail/blocks would say 95% here too by coincidence, so // use a case where they diverge: 1000 total, 500 free, 200 available. expect(percentUsed(1000, 500, 200)).toBe(71); // 500 used of 700 usable }); test('a full filesystem reports 100', () => { expect(percentUsed(1000, 0, 0)).toBe(100); }); test('a degenerate zero-block filesystem does not divide by zero', () => { expect(percentUsed(0, 0, 0)).toBe(0); }); }); describe('parseDfOutput', () => { test('parses real df -P output', () => { const out = [ 'Filesystem 1024-blocks Used Available Capacity Mounted on', '/dev/mmcblk0p2 120699413 11534336 106954752 10% /', ].join('\n'); expect(parseDfOutput(out)).toEqual({ usedPercent: 10, availableBytes: 106_954_752 * 1024, }); }); test('parses a nearly-full filesystem', () => { const out = [ 'Filesystem 1024-blocks Used Available Capacity Mounted on', '/dev/sda1 41284928 39220684 966140 98% /', ].join('\n'); expect(parseDfOutput(out)?.usedPercent).toBe(98); }); test('returns null rather than guessing on unusable output', () => { expect(parseDfOutput('')).toBeNull(); expect(parseDfOutput('Filesystem 1024-blocks Used Available Capacity Mounted on')).toBeNull(); expect(parseDfOutput('header\ntoo few fields')).toBeNull(); expect(parseDfOutput('header\n/dev/sda1 a b c d /')).toBeNull(); }); test('tolerates leading whitespace and extra columns', () => { const out = ' Filesystem 1024-blocks Used Available Capacity Mounted on\n /dev/sda1 100 50 40 56% / extra'; expect(parseDfOutput(out)?.usedPercent).toBe(56); }); }); describe('diskProbeTargets (celilo#1133)', () => { // The two boxes that actually failed, on one night, in one fleet. // // vmid 204 celilo-registry 20G, 100% full, 4 KB free. Publishes returned // `Internal Server Error`; five release attempts died. Reads kept // working, so health, the sparse index and `module search` all // stayed green. // vmid 206 git.celilo.computer 40G, 100% full. The builder's runner logged // `failed to fetch task ... database or disk is full` every two // seconds and claimed nothing for ~13 hours. 33 GB of it was 785 // unpruned Actions bundles in repo-archive. It presented as a DEAD // RUNNER; the runner was healthy, registered and polling. // // Both are container_service instances. Both were invisible to a probe that // walked `listMachines()`, because a `machine` is an operator-pre-provisioned // box and neither of these is one. Two failures, two disguises, one blind spot. const registry = { hostname: 'celilo-registry', ipv4Address: '203.0.113.4', vmid: 204 }; const forge = { hostname: 'forgejo', ipv4Address: '203.0.113.6', vmid: 206 }; const machine = { hostname: 'iot', ipAddress: '203.0.113.5', sshUser: 'root' }; // THE regression, asserted on the two vmids that failed rather than on // invented ones. Scheduling the monitor would NOT have caught either: the // check would have gone on reporting all-clear about filesystems it never // looked at, which is worse than not having the check at all. test('includes the container_service instances that filled, not just the machine pool', () => { const targets = diskProbeTargets([machine], [registry, forge]); expect(targets.map((t) => t.hostname).sort()).toEqual(['celilo-registry', 'forgejo', 'iot']); }); test('a container is probed as root, the user celilo already deploys to it as', () => { const [target] = diskProbeTargets([], [registry]); expect(target).toEqual({ hostname: 'celilo-registry', ipAddress: '203.0.113.4', sshUser: 'root', }); }); test('a machine-pool placement contributes nothing — no vmid, already covered', () => { const onMachine = { hostname: 'iot', ipv4Address: '203.0.113.5', vmid: null }; expect(diskProbeTargets([machine], [onMachine])).toEqual([machine]); }); test('co-hosted modules share one filesystem and so produce one target', () => { const caddy = { hostname: 'shared', ipv4Address: '203.0.113.9', vmid: 210 }; const authentik = { hostname: 'shared', ipv4Address: '203.0.113.9', vmid: 210 }; expect(diskProbeTargets([], [caddy, authentik]).length).toBe(1); }); test('the machine pool wins on a shared address, keeping its configured ssh user', () => { const asMachine = { hostname: 'vps', ipAddress: '203.0.113.4', sshUser: 'admin' }; const [target] = diskProbeTargets([asMachine], [registry]); expect(target.sshUser).toBe('admin'); }); test('a system with no address is skipped rather than probed blindly', () => { expect(diskProbeTargets([], [{ hostname: 'x', ipv4Address: '', vmid: 7 }])).toEqual([]); }); });