/** * Suppression — deciding which alerts explain other alerts. * * When a machine drops off the network, every module on it fails its health * check. Reporting all of them is not thoroughness, it is six pages at 3am for * one fact. Suppression collapses that to the page that names the cause. * * The edges are DERIVED from deployment state, never configured. celilo already * records which modules run on which systems, in which zones, and which modules * provide zone-scoped capabilities; asking an operator to restate that would * guarantee it drifts out of date, and a stale suppression graph is worse than * none — it hides real failures. * * zone ─ (a module providing a zone-scoped capability, e.g. dns_internal) * └── system ─ (builtin:machines_reachable/machine:) * └── module ─ (module:) * └── check item ─ (module:/check:) * * See openspec/changes/add-alerting/design.md D7. */ import { builtinAlertKey, moduleAlertKey, parseAlertKey } from './keys'; /** The audit category whose findings represent an unreachable system. */ export const MACHINES_REACHABLE_CHECK = 'machines_reachable'; export interface ModuleSystemRow { moduleId: string; /** User-facing hostname — the identifier a machine alert is keyed by. */ hostname: string; zone: string; infraType: 'machine' | 'container_service'; } export interface ZoneCapabilityProviderRow { moduleId: string; capabilityName: string; /** Zones this capability is scoped to. Empty means zone-agnostic. */ zones: string[]; } export interface SuppressionTopology { moduleSystems: ModuleSystemRow[]; zoneProviders: ZoneCapabilityProviderRow[]; } /** The machine-level key for a system, by its user-facing hostname. */ export function machineAlertKey(hostname: string): string { return builtinAlertKey(MACHINES_REACHABLE_CHECK, 'machine', hostname); } /** * The keys that, if firing, would explain `key` — nearest ancestor first. * * Order matters: the nearest firing ancestor is the one reported as the cause, * and "caddy's hook could not run" is a better explanation than "something in * the dmz is broken" when both are true. */ export function ancestorKeysFor(key: string, topology: SuppressionTopology): string[] { const parsed = parseAlertKey(key); // A built-in alert (an unreachable machine, a coverage gap) is already a // root cause in this model. Nothing above it explains it. if (!parsed || parsed.source !== 'module') return []; const ancestors: string[] = []; const moduleId = parsed.moduleId; // 1. The module-level key owns its check items. if (parsed.check) ancestors.push(moduleAlertKey(moduleId)); const systems = topology.moduleSystems.filter((s) => s.moduleId === moduleId); // 2. The systems the module runs on. // // Only machine-pool systems: `machines_reachable` probes the machine pool, // so a container_service instance has no corresponding alert to be suppressed // by. A module hosted only in containers therefore gets no system-level // suppression — a real gap, not an oversight, and one that wants a // container-reachability check before it can close. for (const system of systems) { if (system.infraType === 'machine') ancestors.push(machineAlertKey(system.hostname)); } // 3. Modules providing a zone-scoped capability covering a zone this module // sits in — a dead resolver or firewall explains everything behind it. const zones = new Set(systems.map((s) => s.zone)); for (const provider of topology.zoneProviders) { // A provider never suppresses itself through its own zone edge. Without // this, a resolver whose health check fails BECAUSE dns is down would // silence its own alert, and the one thing naming the cause disappears. if (provider.moduleId === moduleId) continue; if (provider.zones.some((zone) => zones.has(zone))) { ancestors.push(moduleAlertKey(provider.moduleId)); } } return ancestors; } export interface SuppressorLookup { key: string; /** Keys of alerts currently firing (candidate suppressors). */ firingKeys: ReadonlySet; /** False for monitors watching the alerting system itself. */ suppressible: boolean; /** Modules currently inside a deploy window. */ modulesInDeployWindow: ReadonlySet; /** * Modules currently PAUSED. A pause is a deliberate quiescing, so its alerts * are explained by the pause itself (openspec/changes/module-pause-lifecycle, * task 2.3) — same mechanism as a deploy window, with the pause as the source * instead of an ancestor alert. */ pausedModules: ReadonlySet; topology: SuppressionTopology; } export type Suppressor = | { kind: 'alert'; key: string } | { kind: 'deploy_window'; moduleId: string } | { kind: 'paused'; moduleId: string }; /** * Find what is suppressing `key`, or null if it should be reported. * * Evaluated at NOTIFY time rather than when the alert fires: monitors do not * run in a guaranteed order, so a module's check can fail seconds before the * machine check that explains it. Deciding at fire time pages for the symptom * moments before the cause arrives. */ export function findSuppressor(lookup: SuppressorLookup): Suppressor | null { // A self-monitor is never suppressed. A cascading failure must not silence // the component reporting the cascade. if (!lookup.suppressible) return null; const parsed = parseAlertKey(lookup.key); // A pause is checked before a deploy window because it is the longer-lived // and more consequential explanation: a paused module may also be inside a // deploy window (unpause redeploys), and "paused" is the fact the operator // needs to see. Attributed to the pause rather than suppressed anonymously — // silently dropping the alert is what turns a pause into an invisible outage. if (parsed?.source === 'module' && lookup.pausedModules.has(parsed.moduleId)) { return { kind: 'paused', moduleId: parsed.moduleId }; } // A deploy is the same mechanism with a window as the source instead of an // ancestor alert — which is why deploy auto-silencing is not a second feature. if (parsed?.source === 'module' && lookup.modulesInDeployWindow.has(parsed.moduleId)) { return { kind: 'deploy_window', moduleId: parsed.moduleId }; } for (const ancestor of ancestorKeysFor(lookup.key, lookup.topology)) { if (lookup.firingKeys.has(ancestor)) return { kind: 'alert', key: ancestor }; } return null; }