/** * Machines-reachable check. * * For each machine in the pool, attempts a quick SSH probe (5s * timeout, BatchMode=yes so it never hangs on a password prompt). * Catches: * * - VPS terminated or paused. * - Raspberry Pi powered off / unplugged. * - SSH key rotated; old key no longer works. * - Network partition (machine on a zone the celilo host can't * reach right now). * * Severity is `drift` — these are operational, not architectural. * The audit consumes pre-computed probe results so it stays unit- * testable without a live SSH client. */ import type { DriftFinding } from './types'; export interface MachineReachableResult { /** * User-facing hostname, and the identifier every finding here is keyed by. * * NOT the machine's UUID. Suppression resolves a machine's ancestor key from * its hostname (`machineAlertKey` in alerting/suppression.ts), so a finding * subjected on the UUID produces an alert key suppression can never match — * an unreachable machine then suppresses nothing and every module on it pages * independently, which is the cascade suppression exists to prevent. That was * celilo#596, filed against this check and fixed here; `disk-space.ts` cites * it as the reason it keys on hostname too. * * The UUID used to be carried alongside as `id`. It is deleted rather than * left unused (Rule 3.9): its only reader was the defect, and a field kept * "just in case" is what the next subject line would reach for. */ hostname: string; ipAddress: string; /** True if SSH probe succeeded. */ reachable: boolean; /** Short description of failure when `!reachable`. */ message?: string; } export interface MachinesReachableAuditDeps { results: MachineReachableResult[]; } export async function auditMachinesReachable( deps: MachinesReachableAuditDeps, ): Promise { const findings: DriftFinding[] = []; // Collapse "all machines unreachable" → one finding pointing at the // network/SSH-key/celilo-host root cause, not N identical lines. const failed = deps.results.filter((r) => !r.reachable); if (failed.length > 1 && failed.length === deps.results.length) { findings.push({ category: 'machines_reachable', severity: 'drift', code: 'all_machines_unreachable', message: `All ${failed.length} machines unreachable from this host`, details: 'Every machine in the pool failed the SSH probe. Likely a\n' + 'network or SSH-key issue on the celilo host, not per-\n' + 'machine failure.', remediation: [ 'Check from this host:', ' - Network: can you ping the machines directly?', ' - SSH key: ls ~/.ssh/ (the key celilo uses to deploy)', ' - DNS: do machine hostnames resolve?', ].join('\n'), actionable: false, subject: 'system', }); return findings; } for (const r of failed) { findings.push({ category: 'machines_reachable', severity: 'drift', code: 'machine_unreachable', message: `${r.hostname} (${r.ipAddress}): SSH unreachable`, details: r.message, remediation: [ 'Verify the machine is powered on and reachable on the', 'network. If the SSH key was rotated, re-add the machine:', ` celilo machine remove ${r.hostname}`, ' celilo machine add # (re-run the wizard)', ].join('\n'), // Multi-step / interactive; not a one-shot. actionable: false, // Hostname, so `machineAlertKey` can match this — see the note on // MachineReachableResult.hostname (celilo#596). subject: r.hostname, }); } return findings; }