/** * Transport-reads check — is celilo still able to READ replies? * * Every other signal that a notification transport is working proves the wrong * half. `send` succeeding proves outbound. A daemon answering its API proves a * socket. Neither can see the return leg fail, and for a week none of them did: * six alert tokens were issued, zero were consumed, and nothing anywhere was * red (#501). * * This is the check that would have caught it. It asserts the contract — * celilo can read this transport — using first-hand evidence rather than a * proxy: the recorded outcome of the reads the poller already performs every * few minutes. * * WHY STALENESS IS A SOUND SIGNAL HERE, and it rests on one decision: * an empty read is recorded as a SUCCESS. A transport nobody has replied on * still records a successful read on every poll. So "no successful read in a * while" cannot mean "quiet" — it can only mean the reads stopped happening or * stopped working. Had zero-messages been recorded as failure, this check would * page every time an operator had a peaceful afternoon, and would be turned off * within a week. * * Deliberately consumes pre-computed state rather than reading the store * itself, so it stays unit-testable and cannot become a second thing that * performs reads. A check that drained the queue to find out whether the queue * could be drained would eat the acknowledgement it was protecting (#541). */ import type { TransportReadStatus } from '../alerting/read-records'; import type { DriftFinding } from './types'; export type { TransportReadStatus }; export interface TransportReadsAuditDeps { /** One entry per transport that has at least one route pointing at it. */ statuses: TransportReadStatus[]; now: Date; /** * How long without a successful read before it counts as drift. * * The poller runs every five minutes, so this is a multiple of that rather * than a guess: it has to absorb a missed tick, a slow sweep, and a restart * without crying wolf, while still catching a transport that has genuinely * stopped being readable. */ staleAfterMs: number; } const CATEGORY = 'transport_reads' as const; function describeAge(ms: number): string { const mins = Math.floor(ms / 60_000); if (mins < 60) return `${mins}m`; const hours = Math.floor(mins / 60); return hours < 48 ? `${hours}h` : `${Math.floor(hours / 24)}d`; } export async function auditTransportReads(deps: TransportReadsAuditDeps): Promise { const findings: DriftFinding[] = []; for (const status of deps.statuses) { const id = status.transportModuleId; // Nothing recorded at all. Either the poller has never run, or this // transport was added and never polled. Both mean no reply from here has // ever been collectable, which is worth saying out loud rather than // treating an empty store as "fine so far". if (!status.last) { findings.push({ category: CATEGORY, severity: 'drift', code: 'transport_never_polled', message: `${id}: celilo has never recorded a read attempt`, details: 'No inbound read has been attempted for this transport, so a\n' + 'reply sent to it would not be collected. This is the state a\n' + 'newly-added transport is in until the first poll runs.', remediation: 'celilo alerts poll', actionable: true, subject: id, }); continue; } // A transport with no `receive` is unidirectional BY DESIGN — pages go out, // replies were never possible, and the route's can_ack already records // that. Flagging it would be flagging a working system. if (status.last.outcome === 'unidirectional') continue; if (!status.last.lastSuccessAt) { findings.push({ category: CATEGORY, severity: 'drift', code: 'transport_never_read', message: `${id}: no read has ever SUCCEEDED`, details: `Reads have been attempted — the most recent was ${status.last.outcome}${status.last.error ? ` (${status.last.error})` : ''} — but none has\never succeeded. Acknowledgements sent to this transport cannot be\ncollected, and outbound paging will keep working, so nothing else\nwill report this.`, remediation: `celilo module journal ${id}`, actionable: true, subject: id, }); continue; } const age = deps.now.getTime() - new Date(status.last.lastSuccessAt).getTime(); if (age > deps.staleAfterMs) { findings.push({ category: CATEGORY, severity: 'drift', code: 'transport_reads_stale', message: `${id}: last successful read ${describeAge(age)} ago`, details: `The most recent attempt was ${status.last.outcome}${status.last.error ? `: ${status.last.error}` : ''}.\nAn empty read still counts as a success, so this is not "nobody\nreplied" — reads are either not happening or not working, and a\nreply sent now would not be collected.`, remediation: `celilo module journal ${id}`, actionable: true, subject: id, }); } } return findings; }