/** * `celilo alerts poll` — read replies off every bidirectional transport once. * * Runs far more often than the five-minute sweep: "I texted ack and nothing * happened" is the experience this exists to prevent. Registered on * `timer.tick.5m` like the sweep, but also safe to run by hand or from a * tighter loop. * * The receive cursor is persisted in system config rather than memory, so a * restart resumes where it left off instead of re-reading or skipping. */ import { eq } from 'drizzle-orm'; import { getEventBusPath } from '../../config/paths'; import { getDb } from '../../db/client'; import { systemConfig } from '../../db/schema'; import { makeReceiver, pollInbound } from '../../services/alerting/inbound-poller'; import { startNotificationResponder } from '../../services/alerting/notification-responder'; import { listRoutes } from '../../services/alerting/people'; import { readLastRead, writeLastRead } from '../../services/alerting/read-records'; import { loadNotificationTransport } from '../../services/alerting/transport-loader'; import type { CommandResult } from '../types'; /** How long a reply token stays usable. A deploy may wait overnight. */ const INTERVIEW_TTL_MS = 24 * 60 * 60_000; const CURSOR_PREFIX = 'alerting.inbound_cursor.'; function readCursor(db: ReturnType, transportModuleId: string): string | null { const key = `${CURSOR_PREFIX}${transportModuleId}`; const row = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get(); return row?.value ?? null; } function writeCursor( db: ReturnType, transportModuleId: string, cursor: string | null, ): void { const key = `${CURSOR_PREFIX}${transportModuleId}`; if (cursor === null) return; const existing = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get(); if (existing) { db.update(systemConfig).set({ value: cursor }).where(eq(systemConfig.key, key)).run(); } else { db.insert(systemConfig) .values({ key, value: cursor, description: `Inbound receive cursor for ${transportModuleId}`, }) .run(); } } /** * " (last read OK 3h ago)" or " (never read successfully)". * * Appended to a read failure because the failure alone does not say how bad it * is. One failed poll is a blip; a transport that has not been readable since * Tuesday is an outage nobody was told about, and those two produced identical * output until this record existed. */ function sinceLastSuccess(db: ReturnType, transportModuleId: string): string { const last = readLastRead(db, transportModuleId)?.lastSuccessAt; if (!last) return ' (never read successfully)'; const ms = Date.now() - new Date(last).getTime(); const mins = Math.floor(ms / 60_000); if (mins < 60) return ` (last read OK ${mins}m ago)`; const hours = Math.floor(mins / 60); return hours < 48 ? ` (last read OK ${hours}h ago)` : ` (last read OK ${Math.floor(hours / 24)}d ago)`; } export async function handleAlertsPoll( flags: Record = {}, ): Promise { const db = getDb(); const askErrors: string[] = []; // `--ask-after 0` asks immediately. Useful for an operator who wants a // question escalated now, and for tests that would otherwise wait out the // grace. const raw = flags['ask-after']; const askAfterSeconds = typeof raw === 'string' ? Number.parseInt(raw, 10) : null; if (raw !== undefined && (askAfterSeconds === null || Number.isNaN(askAfterSeconds))) { return { success: false, error: '--ask-after expects a number of seconds' }; } // The responder both DELIVERS unanswered questions and publishes the // answers. Both halves belong here: a question delivered by a process that // then exits is still answerable, because what was asked lives in the // deliveries table rather than in that process's memory. const responder = startNotificationResponder({ db, busDbPath: getEventBusPath(), routes: listRoutes(db).filter((route) => route.enabled && route.canAck), transportFor: (route) => loadNotificationTransport(db, route.transportModuleId), // A CLI invocation is never the terminal responder for someone else's // deploy — that responder attaches to its own deploy's stdin. hasTty: () => false, now: () => new Date(), ttlMs: INTERVIEW_TTL_MS, askAfterMs: askAfterSeconds === null ? undefined : askAfterSeconds * 1000, onSendFailed: (eventType, error) => askErrors.push(`${eventType}: ${error}`), }); let ask = { asked: 0, failed: 0 }; try { ask = await responder.poll(); } catch { // A bus that cannot be opened must not stop alert replies being read — // those are the 3am path. } const report = await pollInbound(db, { receiveFrom: makeReceiver(db), readCursor: (t) => readCursor(db, t), writeCursor: (t, c) => writeCursor(db, t, c), recordRead: (t, r) => writeLastRead(db, t, r), now: () => new Date(), transportFor: (route) => loadNotificationTransport(db, route.transportModuleId), answerInterview: (eventId, value) => responder.answer(eventId, value), }); responder.stop(); const parts = [ `${report.transportsPolled} transport(s)`, `${report.messagesRead} message(s)`, `${report.acked} acked`, ]; if (report.unheard.length > 0) parts.push(`${report.unheard.length} NOT HEARD`); if (report.broadcast > 0) parts.push(`${report.broadcast} told someone has it`); if (report.answered > 0) parts.push(`${report.answered} deploy question(s) answered`); if (ask.asked > 0) parts.push(`${ask.asked} question(s) asked`); // Never silent: a deploy blocked on a question nobody received is // indistinguishable from a deploy that is merely slow. if (ask.failed > 0) parts.push(`${ask.failed} question(s) COULD NOT BE DELIVERED`); // The summary line counts; these lines say WHY. Without them "0 message(s)" // means both "nobody replied" and "the reply was read and thrown away", and // an operator has no way to tell which — the ambiguity that made the Signal // ack path take a week to diagnose. const detail = [ ...report.failures.map( (f) => `${f.transportModuleId} COULD NOT BE READ: ${f.error}${sinceLastSuccess(db, f.transportModuleId)}`, ), ...report.unheard.map((u) => `not heard from ${u.senderAddress}: ${u.reason}`), ...askErrors, ]; const message = `inbound poll: ${parts.join(', ')}`; return { success: true, message: detail.length > 0 ? `${message}\n ${detail.join('\n ')}` : message, }; }