/** * Reachability probe for the machine pool. * * One implementation, deliberately. This SSH probe previously existed twice — * once in `alerting/builtin-source.ts` for the monitor sweep and once in * `cli/commands/system-audit.ts` for `celilo system audit` — as identical * copy-pasted blocks. Both carried the same bug, and fixing one would have left * the other reporting the management server as unreachable forever. */ import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import type { MachineReachableResult } from './audit/machines-reachable'; import { listMachines } from './machine-pool'; import { LOCAL_MACHINE_IP } from './ssh-key-manager'; const execFileAsync = promisify(execFile); /** * Probe every machine in the pool. * * `BatchMode=yes` prevents a password prompt from hanging the probe forever, * and `ConnectTimeout` bounds the wait on an unresponsive host — the exact * condition this check exists to detect must not be the one that wedges it. * * The local management box is reported reachable WITHOUT probing it. celilo * runs there as the `celilo` user and deliberately does not materialize an SSH * key for itself, so `ssh root@127.0.0.1` fails with `Permission denied * (publickey)` on a perfectly healthy host. Left unhandled that produced a * permanently firing `machines_reachable` alert against celilo-mgr — and since * that monitor is unsuppressible by design, nothing could explain it away. * * The check is also meaningless there: if the box running this code were * unreachable, this code would not be running. */ export async function probeMachines(): Promise { const machines = await listMachines(); return Promise.all( machines.map(async (m): Promise => { if (m.ipAddress === LOCAL_MACHINE_IP) { return { hostname: m.hostname, ipAddress: m.ipAddress, reachable: true }; } try { await execFileAsync( 'ssh', [ '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=5', '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', `${m.sshUser}@${m.ipAddress}`, 'true', ], { timeout: 8000 }, ); return { hostname: m.hostname, ipAddress: m.ipAddress, reachable: true }; } catch (err) { const e = err as { stderr?: string; message?: string }; return { hostname: m.hostname, ipAddress: m.ipAddress, reachable: false, message: e.stderr?.trim() || e.message || 'SSH probe failed', }; } }), ); }