/** * The narrow reads the web console polls. * * Separate from the CLI's own reads for a measured reason: `celilo module list * --json` returns 156 KB for 23 modules because it embeds every module's * `manifestData` blob. That is the right payload for a human debugging one * module and the wrong one for a loop that runs every few seconds, so the * console gets a projection with the manifest left out and the two derived * facts it actually renders (observed health, backup freshness) already * computed. * * Reads only. Nothing here writes, and the console's API principal is granted * only read ops, so a write added here would fail at the boundary rather than * succeed quietly. */ import { FIREWALL_CAPABILITY_NAME, orderFirewallChain } from '@celilo/capabilities'; import { desc, eq } from 'drizzle-orm'; import { z } from 'zod'; import type { DbClient } from '../db/client'; import { NETWORK_ZONES, backups, capabilities, moduleSystems, modules, monitors, } from '../db/schema'; import { loadObservedHealthDetail } from '../services/alerting/observed-health'; import { loadBackupAuditInfo } from '../services/audit/backup-source'; import { backupStaleThresholdMs, moduleHasBackupHook } from '../services/audit/backups'; import { effectiveBackupSchedule } from '../services/backup-schedule'; import { listCapabilityBindings } from '../services/capability-bindings'; import type { ConsumedCapabilities } from '../services/consumer-cleanup'; import type { DelegationChain } from './closure'; /** * Just the capability lists, validated at the boundary. * * `modules.manifestData` is opaque JSON in the database, which is a trust * boundary even though celilo wrote it: rows outlive the code that wrote them, * and an older celilo's row is exactly the case a clean-database test can never * produce. `.catch` rather than `.parse` because one unreadable manifest must * end that branch of the walk, not blank the whole topology. */ const CONSUMED_CAPABILITIES = z .object({ requires: z .object({ capabilities: z.array(z.object({ name: z.string() })).optional() }) .optional(), optional: z .object({ capabilities: z.array(z.object({ name: z.string() })).optional() }) .optional(), }) .catch({}); export interface ConsoleSystem { hostname: string; address: string; zone: string; } export interface ConsoleModule { id: string; version: string; state: string; health: { cell: string; monitored: boolean; firingCount: number; suppressed: boolean }; systems: ConsoleSystem[]; /** * Epoch MILLISECONDS of the newest COMPLETED backup, or null if never. * * Milliseconds because every other instant that crosses this boundary is in * milliseconds, and a single field in seconds is a factor of a thousand that * no type catches. The console renders ages, so the failure mode is a * plausible wrong number rather than a crash: the alerts route computed one * age in the wrong unit and rendered every alert as `0m`, a fleet where * nothing had been wrong for over a minute. */ lastBackupAt: number | null; lastBackupFailed: boolean; /** * Nothing is checking this module. * * No enabled monitor, or one whose interval is manual, or one that has never * run. Distinct from healthy: an unwatched module's silence is not evidence * that it is fine, and `ok` and `unwatched` look identical on a dashboard * that only tracks health. */ unwatched: boolean; /** * It IS checked, on a schedule, and its monitor has no escalation policy. * * The alert is raised and reaches nobody. Worse than unwatched in one * respect, because it looks monitored, so it is reported separately rather * than folded into the health cell. */ pagesNobody: boolean; /** * The newest successful backup is older than this module's cadence allows. * * Resolved HERE, through the same accessor the backup sweep and the drift * audit use, so the three cannot disagree about what a module's cadence is. * False for a module with no `on_backup` hook: there is nothing to run, so it * is not overdue. */ backupStale: boolean; } export interface ConsoleStatus { /** * The canonical zone order, most exposed first. Served rather than compiled * into the console so a zone added to `NETWORK_ZONES` gets a band without a * console release. */ zones: string[]; modules: ConsoleModule[]; } /** * The dashboard's single poll: the zone order, and every module with the two * derived columns the roster shows. * * One call rather than three because a poll that changes nothing should cost * one round trip, and because the topology, the roster and the backup column * all read the same rows. */ export function consoleStatus(db: DbClient): ConsoleStatus { const health = loadObservedHealthDetail(db); const systemsByModule = new Map(); for (const row of db.select().from(moduleSystems).all()) { const list = systemsByModule.get(row.moduleId) ?? []; list.push({ hostname: row.hostname, address: row.ipv4Address, zone: row.zone }); systemsByModule.set(row.moduleId, list); } const backupByModule = latestBackupByModule(db); const watch = watchFactsByModule(db); const overdue = staleBackupModules(db); const rows = db .select({ id: modules.id, version: modules.version, state: modules.state }) .from(modules) .all(); return { zones: [...NETWORK_ZONES], modules: rows .map((row): ConsoleModule => { const backup = backupByModule.get(row.id); return { id: row.id, version: row.version, state: row.state, health: health.get(row.id) ?? { // A module that is not deployed has nothing to observe. Saying // "not observed" here would report undeployed modules as a finding. cell: 'not deployed', monitored: false, firingCount: 0, suppressed: false, }, systems: systemsByModule.get(row.id) ?? [], lastBackupAt: backup?.lastSuccessAt ?? null, lastBackupFailed: backup?.lastFailed ?? false, // Absent from the map means no enabled monitor at all, which is the // loudest form of unwatched rather than the quietest. unwatched: watch.get(row.id)?.unwatched ?? true, pagesNobody: watch.get(row.id)?.pagesNobody ?? false, backupStale: overdue.has(row.id), }; }) .sort((a, b) => a.id.localeCompare(b.id)), }; } /** * What celilo is doing to watch each module, if anything. * * Two facts, kept apart because an operator acts differently on each. A module * nothing checks is invisible. A module that IS checked on a schedule and whose * monitor routes to no escalation policy is worse in one respect: it looks * monitored, the alert gets raised on time, and it reaches nobody. * * `interval manual` and `never run` both count as unwatched. A monitor that * exists and has not executed produces exactly as much evidence as one that * does not exist, and the console must not report the first as coverage. */ interface WatchFacts { unwatched: boolean; pagesNobody: boolean; } function watchFactsByModule(db: DbClient): Map { const result = new Map(); for (const monitor of db.select().from(monitors).where(eq(monitors.enabled, true)).all()) { const running = monitor.intervalMinutes > 0 && monitor.lastRunAt !== null; const existing = result.get(monitor.target); // A module can carry more than one monitor. It is watched if ANY of them // runs, and pages nobody only if every running one lacks a policy. result.set(monitor.target, { unwatched: (existing?.unwatched ?? true) && !running, pagesNobody: (existing?.pagesNobody ?? false) || (running && monitor.escalationPolicyId === null), }); } return result; } /** * Modules whose newest successful backup is older than their cadence allows. * * Resolved through `effectiveBackupSchedule` and `backupStaleThresholdMs`, the * same two functions the drift audit uses. The cadence in force comes from the * operator's override and then the manifest and defaults to daily when neither * says anything, so a console that worked it out again would be a second * definition of "overdue" and would disagree the first time an override was * set. * * A module with no `on_backup` hook is never stale. There is nothing to run. */ function staleBackupModules(db: DbClient): Set { const stale = new Set(); const now = Date.now(); for (const info of loadBackupAuditInfo(db)) { if (!moduleHasBackupHook(info.manifest)) continue; const cadence = effectiveBackupSchedule(info.manifest, info.scheduleOverride); const threshold = backupStaleThresholdMs(cadence); // `manual` has no threshold. Opting out is a decision, not a fault. if (threshold === null) continue; const last = info.lastSuccessfulBackupAt; if (last === null || now - last > threshold) stale.add(info.id); } return stale; } interface BackupFacts { lastSuccessAt: number | null; lastFailed: boolean; } /** * Newest completed backup per module, and whether the most recent ATTEMPT * failed. * * Both, because they answer different questions and the console shows both. A * module can hold a fresh successful backup and still have failed last night, * and a roster that showed only the success date would call that healthy. */ function latestBackupByModule(db: DbClient): Map { const result = new Map(); const rows = db .select({ moduleId: backups.moduleId, status: backups.status, startedAt: backups.startedAt, completedAt: backups.completedAt, }) .from(backups) .orderBy(desc(backups.startedAt)) .all(); for (const row of rows) { if (!row.moduleId) continue; // a system-state backup belongs to no module const existing = result.get(row.moduleId); if (!existing) { result.set(row.moduleId, { lastSuccessAt: row.status === 'completed' && row.completedAt ? epochMillis(row.completedAt) : null, lastFailed: row.status === 'failed', }); continue; } if (existing.lastSuccessAt === null && row.status === 'completed' && row.completedAt) { existing.lastSuccessAt = epochMillis(row.completedAt); } } return result; } /** * Milliseconds, deliberately, and named so nobody has to guess. * * This was `epochSeconds`, and the protocol on the other side of the boundary * documents the same field as milliseconds. The two never met because the * console server does not exist yet, so the factor of a thousand sat there * looking like working code. It is the exact shape of the bug that rendered * every alert as `0m`. */ function epochMillis(value: Date): number { return value.getTime(); } /** * Which providers a module has actually called into, by capability. * * The difference between a manifest and a fleet. A manifest says what a module * CAN consume; this says what it resolved to and reached for. `tango-nexus` * declares four optional capabilities and is bound to one, and until * celilo#1072 landed there was no way to tell those apart. * * The console previously inferred this: a REQUIRED capability of a deployed * module must have bound, because the deploy would have failed otherwise. That * was the only sound inference available and it is now an approximation, so it * is gone rather than kept as a fallback (Rule 3.9). A module with no rows here * has called into nothing, which is a real answer and not a missing one. */ export function loadBindings(db: DbClient, moduleId: string): Map { return new Map( listCapabilityBindings(db, moduleId).map((b) => [b.capabilityName, b.providerModuleId]), ); } /** * Everything the closure walk needs, read in ONE place. * * The chains are in here rather than beside it deliberately. A separate loader * a caller has to remember is a caller that forgets, and the answer when it * forgets is an empty chain list that reads exactly like a fleet with no * delegation. There is one call site, and it cannot get a partial set. */ export function loadClosureInputs(db: DbClient): { manifests: Map; providerStates: { moduleId: string; state: string }[]; chains: DelegationChain[]; } { return { manifests: new Map( db .select({ id: modules.id, manifestData: modules.manifestData }) .from(modules) .all() .map((row) => [row.id, CONSUMED_CAPABILITIES.parse(row.manifestData)] as const), ), providerStates: db.select({ moduleId: modules.id, state: modules.state }).from(modules).all(), chains: loadFirewallChain(db), }; } /** * The `firewall` providers in delegation order, as at most one chain. * * At most one because `firewall` is the only capability celilo chains, and * `orderFirewallChain` — the SAME function the hook loader wires the live * providers with — is what decides the order. A fleet with one firewall, or * with none that reaches the internet, yields no chain at all. */ export function loadFirewallChain(db: DbClient): DelegationChain[] { const providers = db .select({ moduleId: capabilities.moduleId, data: capabilities.data }) .from(capabilities) .where(eq(capabilities.capabilityName, FIREWALL_CAPABILITY_NAME)) .all(); const ordered = orderFirewallChain(providers).map((provider) => provider.moduleId); return ordered.length === 0 ? [] : [{ capability: FIREWALL_CAPABILITY_NAME, moduleIds: ordered }]; } /** A module's row, or undefined. Used to reject a closure request for a module that is gone. */ export function moduleExists(db: DbClient, moduleId: string): boolean { return ( db.select({ id: modules.id }).from(modules).where(eq(modules.id, moduleId)).all().length > 0 ); }