/** * Public-DNS reachability check. * * Every check celilo had reported healthy throughout celilo#626, during which * five public names — including the apt repo and the module registry — resolved * to an address that answered nothing for nine days. None of them was wrong: * they all looked from *inside*, where the split-horizon resolver deliberately * answers with an address that is reachable in-zone. celilo simply had no * notion of a check whose vantage point is outside itself. * * This is that check. For every name in the DNS registration ledger it asks an * OFF-FLEET resolver what the internet resolves, and compares that against the * address the fleet actually appears to come from according to an independent * echo service. * * Three properties are load-bearing, and each exists because of a specific way * this check could quietly become useless: * * - **The resolver must not be the fleet's.** A `public_dns` check that used * the configured resolver would pass forever, exactly like caddy's `dig` * does today. That is the original bug one layer up, so the probe asserts * it rather than leaving it to a comment (see `public-dns-probe.ts`). * - **The expectation must not come from the registrar.** Comparing what was * published against what we asked to publish is self-agreement — and worse * than useless here, because Namecheap's DDNS API returns `ErrCount 0` with * the requested address echoed back for a `www` update it silently does not * apply (design.md D3). * - **Missing evidence is not a pass.** The one genuine external probe the * fleet had (isitup.org, in celilo-website's health check) was itself * unreachable during the outage, and recorded that as *undetermined* with no * check item at all. So a real outage produced complete silence. A single * undetermined result stays quiet — a prober blip must not page — but * consecutive ones become their own finding, distinct from "unreachable". * * Pure over an injected probe and the previous run's evidence counters: no * network, no DB. See design.md D2. */ import type { DriftFinding } from './types'; /** What an off-fleet resolver answered for one name. */ export type PublicResolution = | { kind: 'answer'; ip: string; ttlSeconds: number } /** The resolver answered authoritatively that there is no A record. */ | { kind: 'no_record' } /** The resolver could not be reached, or failed. Not a pass, not a failure. */ | { kind: 'undetermined'; reason: string }; /** The address the fleet appears to come from, per an independent third party. */ export type IngressObservation = | { kind: 'observed'; ip: string } | { kind: 'undetermined'; reason: string }; export interface PublicDnsProbe { /** Named so findings can say where the answer came from, and so a test can assert it is off-fleet. */ readonly resolver: string; /** Named for the same reason — an echo service outage must be attributable. */ readonly echoService: string; observeIngress(): Promise; resolve(fqdn: string): Promise; } /** One ledger name to check. */ export interface PublicDnsRecord { fqdn: string; /** * Claimed by celilo as the companion of a declared name rather than asked * for by a module. Best effort at claim time, so its remediation is the * manual registrar step rather than "fix the registrar and redeploy". */ companion: boolean; /** * When the fleet last asserted this name (the provider's last refresh, or * the registration). Divergence inside one TTL of an assert is propagation, * not a fault — see the hysteresis note below. */ lastAssertedAt: Date; } /** * Per-subject counter of consecutive runs that produced no evidence. Carried * across runs by the caller; the audit itself neither reads nor writes storage. */ export interface PublicDnsEvidence { subject: string; undeterminedRuns: number; } export interface PublicDnsAuditDeps { records: PublicDnsRecord[]; probe: PublicDnsProbe; /** Counters from the previous run. Empty on a first run. */ evidence?: PublicDnsEvidence[]; /** Sink for the updated counters. Omitted in unit tests. */ saveEvidence?: (evidence: PublicDnsEvidence[]) => void; now?: Date; /** Consecutive undetermined runs before the absence of evidence is itself a finding. */ undeterminedThreshold?: number; } /** The echo service's own subject, so its outage is counted like any other. */ const ECHO_SUBJECT = 'system'; const DEFAULT_UNDETERMINED_THRESHOLD = 3; /** * A record that diverges within one TTL of the last assert is still * propagating. Fleet records measure 180s, so alerting on first divergence * would page on every ISP re-lease. Used when the resolver gave no TTL. */ const FALLBACK_TTL_SECONDS = 300; function companionRemediation(fqdn: string): string { const label = fqdn.startsWith('www.') ? 'www' : '@'; return [ `celilo claims ${fqdn} alongside the name the module declared, but the`, 'registrar could not publish it. A DDNS API can update an existing record', 'and not create one, and Namecheap reports success for a `www` update it', 'silently does not apply — so this has to be fixed at the registrar:', '', ` 1. Registrar → DNS for this domain → add an A record with Host="${label}"`, ' (any value — celilo overwrites it), and enable Dynamic DNS.', ' 2. celilo claims it on the next assert; no redeploy needed.', ].join('\n'); } /** * For callers that legitimately have nothing to check — `system update` runs a * partial audit over what it already has and does not reach the network. It * throws rather than answering, so a caller that grows records later cannot * silently keep an inert vantage point. */ export const unusedPublicDnsProbe: PublicDnsProbe = { resolver: '(not used)', echoService: '(not used)', observeIngress() { throw new Error('public_dns probe used by a caller that declared no records'); }, resolve() { throw new Error('public_dns probe used by a caller that declared no records'); }, }; export async function auditPublicDns(deps: PublicDnsAuditDeps): Promise { // No names asserted publicly means nothing to verify — and no reason to // reach the network or advance an undetermined counter. if (deps.records.length === 0) return []; const now = deps.now ?? new Date(); const threshold = deps.undeterminedThreshold ?? DEFAULT_UNDETERMINED_THRESHOLD; const previous = new Map((deps.evidence ?? []).map((e) => [e.subject, e.undeterminedRuns])); const next = new Map(); const findings: DriftFinding[] = []; /** * Count an absence of evidence. The first few are silent — one prober blip * must not page — but they are never forgotten, which is the difference * between "we were not told" and "we could not have been told". */ const countUndetermined = (subject: string): number => { const runs = (previous.get(subject) ?? 0) + 1; next.set(subject, runs); return runs; }; const ingress = await deps.probe.observeIngress(); if (ingress.kind === 'undetermined') { const runs = countUndetermined(ECHO_SUBJECT); // Every name is unverifiable when the expectation is, so the records are // left uncounted rather than each accruing a duplicate of the same outage. if (runs >= threshold) { findings.push({ category: 'public_dns', // The vantage point is gone: nothing was measured this run, and D7 // says that is `unmeasured`, not `drift`. This finding was `drift` // before D7 existed; its own text already said what unmeasured means. severity: 'unmeasured', code: 'public_dns_unverifiable', message: `Public DNS has been unverifiable for ${runs} consecutive checks`, details: `The echo service (${deps.probe.echoService}) could not be reached, so there is no\nexpectation to compare public DNS against. Latest reason: ${ingress.reason}.\n\nThis is NOT a report that the fleet is reachable, and not a report that it\nis unreachable — it is a report that nothing is currently checking. The\noutage this check exists for ran for nine days behind exactly this silence.`, remediation: [ 'Check outbound connectivity from the management host, then:', ` celilo system config set public_dns.echo_url # current: ${deps.probe.echoService}`, ].join('\n'), actionable: false, subject: ECHO_SUBJECT, }); } deps.saveEvidence?.(toEvidence(next)); return findings; } for (const record of deps.records) { const resolution = await deps.probe.resolve(record.fqdn); if (resolution.kind === 'undetermined') { const runs = countUndetermined(record.fqdn); if (runs >= threshold) { findings.push({ category: 'public_dns', // The resolver would not answer for this name, so whether it resolves // publicly is unknown — not fine, not broken. Unmeasured per D7. severity: 'unmeasured', code: 'public_dns_unverifiable', message: `${record.fqdn}: public reachability unverifiable for ${runs} consecutive checks`, details: `${deps.probe.resolver} did not answer for this name. Latest reason: ${resolution.reason}.\nWhether the name resolves publicly is currently unknown — which is not the\nsame as it being fine.`, remediation: `Resolve it by hand to see what the internet gets:\n dig @${deps.probe.resolver} ${record.fqdn} A`, actionable: false, subject: record.fqdn, }); } continue; } if (resolution.kind === 'no_record') { findings.push( record.companion ? { category: 'public_dns', severity: 'drift', code: 'public_dns_companion_unclaimed', message: `${record.fqdn}: not published (companion of a name celilo serves)`, details: 'This name has no public A record, so it does not reach the fleet at all.\n' + 'A working www beside a dead apex (or the reverse) is the same defect as\n' + 'celilo#626 in miniature: publicly broken, locally invisible.', remediation: companionRemediation(record.fqdn), actionable: false, subject: record.fqdn, } : { category: 'public_dns', severity: 'drift', code: 'public_dns_missing', message: `${record.fqdn}: no public A record`, details: `celilo registered this name but ${deps.probe.resolver} resolves no A record for\nit. The registrar reported success; a provider's success response is a claim\nabout an API call, not evidence that the record was published.`, remediation: `Re-assert it, then re-check:\n celilo module run-hook refresh_registrations\n dig @${deps.probe.resolver} ${record.fqdn} A`, actionable: false, subject: record.fqdn, }, ); continue; } if (resolution.ip === ingress.ip) continue; // Hysteresis. A legitimate address change produces a genuinely divergent // public record until the old answer's TTL expires, so a divergence is only // a finding once it has outlived the TTL of the record we last asserted. // Without this the check pages on every ISP re-lease. const ttlSeconds = resolution.ttlSeconds > 0 ? resolution.ttlSeconds : FALLBACK_TTL_SECONDS; const settledAt = record.lastAssertedAt.getTime() + ttlSeconds * 1000; if (now.getTime() < settledAt) continue; findings.push( record.companion ? { category: 'public_dns', severity: 'drift', code: 'public_dns_companion_unclaimed', message: `${record.fqdn}: publicly resolves to ${resolution.ip}, not ${ingress.ip}`, details: `${deps.probe.resolver} serves ${resolution.ip} for this companion name while the fleet is\nreachable at ${ingress.ip} (per ${deps.probe.echoService}). A parked or redirect record is\nthe usual cause: Namecheap renders those as A records the DDNS endpoint\naccepts updates for and does not apply, reporting ErrCount 0 (design.md D3).`, remediation: companionRemediation(record.fqdn), actionable: false, subject: record.fqdn, } : { category: 'public_dns', severity: 'drift', code: 'public_dns_stale', message: `${record.fqdn}: publicly resolves to ${resolution.ip}, not ${ingress.ip}`, details: `${deps.probe.resolver} serves ${resolution.ip} for this name while the fleet is reachable at\n${ingress.ip} (per ${deps.probe.echoService}). The divergence has outlived the record's own\nTTL (${ttlSeconds}s since the last assert), so this is not propagation.\n\nNothing inside the fleet can see this: the split-horizon resolver answers\nwith an address that IS reachable in-zone, which is correct for its purpose\nand says nothing about the public internet.`, remediation: `Re-assert the record, then re-check:\n celilo module run-hook refresh_registrations\n dig @${deps.probe.resolver} ${record.fqdn} A`, actionable: false, subject: record.fqdn, }, ); } deps.saveEvidence?.(toEvidence(next)); return findings; } /** * Only subjects that were undetermined THIS run survive. A subject that * answered has no counter, which is what makes the count "consecutive". */ function toEvidence(next: Map): PublicDnsEvidence[] { return [...next].map(([subject, undeterminedRuns]) => ({ subject, undeterminedRuns })); }