import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; import { DnsRecordDoc } from '../db/documents/classes.dns-record.doc.js'; import { DomainDoc } from '../db/documents/classes.domain.doc.js'; import { EmailDomainDoc } from '../db/documents/classes.email-domain.doc.js'; import type { DcRouter } from '../classes.dcrouter.js'; import type { IEmailDnsRecordIntent, IEmailDomainDnsRevision, IEmailDomainEdgeIdentity, IEmailDomainMxSlot, IEmailDomainOperationError, IEmailDkimMaterial, TEmailDnsVerificationOutcome, } from '../../ts_interfaces/data/email-domain.js'; import type { TDnsRecordType } from '../../ts_interfaces/data/dns-record.js'; import { MailEdgeEligibility, ReleasedRemoteIngressEgressIdentitySource, type IEligibleMailEdge, type IMailEdgeEligibilityResult, } from './classes.mail-edge-eligibility.js'; import { evaluateMailTxtIntent, type IMailDnsTxtPolicyResult, } from './mail-dns-txt-policy.js'; import { applyMailDnsVerificationResult, reconcileMailDnsSingleton, } from './reconcile-mail-dns-singleton.js'; import { projectMailDnsIntentAggregateStatus } from './mail-dns-status.js'; export const MAIL_DNS_SYNC_CREATED_BY = 'mail-dns-reconciler'; export const MAIL_DNS_MANAGED_BY = 'mail-dns-reconciler'; export const MAIL_DNS_RETRY_INTERVAL_MS = 5 * 60_000; export const MAIL_DNS_MAX_RETRY_INTERVAL_MS = 60 * 60_000; export const MAIL_DNS_RECONCILIATION_INTERVAL_MS = 5 * 60_000; export const MAIL_DNS_EDGE_ELIGIBILITY_DEBOUNCE_MS = 250; export const MAIL_DNS_EDGE_ELIGIBILITY_COOLDOWN_MS = 5_000; export const MAIL_DNS_PUBLIC_RESOLVER_OPTIONS = { strategy: 'prefer-udp' as const, allowDohFallback: true, timeoutMs: 5_000, }; export interface IMailDnsVerificationResult { valid: boolean; outcome?: TEmailDnsVerificationOutcome; reason?: string; effectiveValue?: string; } export interface IMailDnsVerifier { verifyRecord(intent: IEmailDnsRecordIntent): Promise; verifyMxSet?( intents: IEmailDnsRecordIntent[], previousValues?: string[], ): Promise; verifyForwardConfirmedIdentity( identity: IEmailDomainEdgeIdentity, ): Promise; evaluateTxtRecords?( intent: IEmailDnsRecordIntent, values: string[], ): Promise; destroy?(): void; } function normalizeHostname(value: string): string { return value.trim().toLowerCase().replace(/\.$/, ''); } function mailEdgeEligibilitySignature(result: IMailEdgeEligibilityResult): string { const edges = result.edges.map((candidate) => { const identities = candidate.identities.map((identity) => ({ family: identity.family, address: identity.address, heloHostname: normalizeHostname(identity.heloHostname), proof: identity.proof, })); identities.sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right))); return { id: candidate.edge.id, hostname: normalizeHostname(candidate.hostname), tags: [...(candidate.edge.tags || [])].sort(), publicIp: candidate.edge.publicIp || '', publicIpV6: candidate.edge.publicIpV6 || '', identities, }; }); edges.sort((left, right) => left.id.localeCompare(right.id)); return JSON.stringify({ supported: result.supported, edges }); } function normalizeMx(value: string): string { const [priority, exchange = ''] = value.trim().split(/\s+/); return `${Number(priority)} ${normalizeHostname(exchange)}`; } function parseMx(value: string): string | undefined { const match = value.trim().match(/^(\d{1,5})\s+([^\s]+)$/); if (!match) return undefined; const priority = Number.parseInt(match[1], 10); const exchange = normalizeHostname(match[2]); if ( priority < 0 || priority > 65_535 || !exchange || exchange.length > 253 || !exchange.split('.').every((label) => ( label.length > 0 && label.length <= 63 && /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i.test(label) )) ) return undefined; return `${priority} ${exchange}`; } export function evaluateMailMxValues( intents: IEmailDnsRecordIntent[], rawValues: string[], previousValues: string[] = [], ): IMailDnsVerificationResult { const parsed = rawValues.map(parseMx); if (parsed.some((value) => value === undefined)) { return { valid: false, outcome: 'invalid', reason: 'MX RRset contains an invalid priority or exchange' }; } const actual = parsed as string[]; const desired = new Set(intents.map((intent) => normalizeMx(intent.value))); const previous = new Set(previousValues.map(normalizeMx)); const counts = new Map(); for (const value of actual) counts.set(value, (counts.get(value) || 0) + 1); const unexpected = actual.filter((value) => !desired.has(value) && !previous.has(value)); if (unexpected.length > 0) { return { valid: false, outcome: 'conflict', reason: `MX RRset contains unexpected value ${unexpected[0]}` }; } if ([...counts.values()].some((count) => count > 1)) { return { valid: false, outcome: 'duplicate', reason: 'MX RRset contains duplicate values' }; } const missing = [...desired].filter((value) => !counts.has(value)); if (missing.length > 0) { return { valid: false, outcome: 'propagation', reason: `MX ${missing[0]} has not propagated` }; } return { valid: true, outcome: 'valid' }; } export class SmartDnsMailVerifier implements IMailDnsVerifier { private client?: plugins.smartdns.dnsClientMod.Smartdns; private getClient(): plugins.smartdns.dnsClientMod.Smartdns { this.client ||= new plugins.smartdns.dnsClientMod.Smartdns(MAIL_DNS_PUBLIC_RESOLVER_OPTIONS); return this.client; } private async queryValues( name: string, type: 'A' | 'AAAA' | 'MX' | 'TXT' | 'PTR', ): Promise< | { status: 'success'; values: string[] } | { status: 'missing'; reason: string } | { status: 'error'; reason: string } > { const result = await this.getClient().queryRecords(name, type); if (result.status === 'success') { return { status: 'success', values: result.records.map((record) => String(record.value).trim()), }; } if (result.status === 'missing') { const evidence = result.attempts.map((attempt) => { if (attempt.status === 'missing') { return `${attempt.transport}:missing:${attempt.missingReason}`; } if (attempt.status === 'error') { return `${attempt.transport}:error:${attempt.error.code}`; } return `${attempt.transport}:success`; }).join(', '); const corroboratedMissing = ( result.attempts.length === 2 && result.attempts[0].transport === 'udp' && result.attempts[0].status === 'missing' && result.attempts[1].transport === 'doh' && result.attempts[1].status === 'missing' ); return corroboratedMissing ? { status: 'missing', reason: `${result.missingReason}; attempts=${evidence}` } : { status: 'error', reason: `unconfirmed ${result.missingReason}; attempts=${evidence || 'none'}` }; } return { status: 'error', reason: `${result.error.code}: ${result.error.message}`, }; } public async verifyRecord(intent: IEmailDnsRecordIntent): Promise { try { const query = await this.queryValues(intent.name, intent.type); if (query.status === 'error') { return { valid: false, outcome: 'lookup-error', reason: `DNS lookup failed: ${query.reason}` }; } if (query.status === 'missing') { return { valid: false, outcome: 'missing', reason: `${intent.type} ${intent.name} is missing (${query.reason})` }; } const values = query.values; if (intent.type === 'MX') { return evaluateMailMxValues([intent], values); } if (intent.type === 'TXT') { return await this.evaluateTxtRecords(intent, values).then((result) => ({ valid: result.outcome === 'valid', ...result, })); } const expectedFamily = intent.type === 'A' ? 4 : 6; if (values.some((value) => plugins.smartnetwork.getIpVersion(value) !== expectedFamily)) { return { valid: false, outcome: 'invalid', reason: `${intent.type} ${intent.name} contains an invalid address`, }; } const normalizedValues = values.map((value) => normalizeHostname(value)); const expected = normalizeHostname(intent.value); const exact = normalizedValues.filter((value) => value === expected); const unexpected = normalizedValues.filter((value) => value !== expected); const outcome: TEmailDnsVerificationOutcome = unexpected.length > 0 ? 'conflict' : exact.length > 1 ? 'duplicate' : exact.length === 0 ? 'propagation' : 'valid'; return { valid: outcome === 'valid', outcome, ...(outcome !== 'valid' ? { reason: `${intent.type} ${intent.name} did not match exactly` } : {}), }; } catch (error: unknown) { return { valid: false, outcome: 'lookup-error', reason: `DNS lookup failed: ${(error as Error).message}` }; } } public async evaluateTxtRecords( intent: IEmailDnsRecordIntent, values: string[], ): Promise { return await evaluateMailTxtIntent(intent, values, { resolveDns: async (name, type) => { const query = await this.queryValues(name, type); if (query.status === 'error') throw new Error(query.reason); return query.status === 'missing' ? [] : query.values; }, }); } public async verifyMxSet( intents: IEmailDnsRecordIntent[], previousValues: string[] = [], ): Promise { if (intents.length === 0) return { valid: false, reason: 'No desired MX records' }; try { const query = await this.queryValues(intents[0].name, 'MX'); if (query.status === 'error') { return { valid: false, outcome: 'lookup-error', reason: `MX lookup failed: ${query.reason}` }; } if (query.status === 'missing') { return { valid: false, outcome: 'missing', reason: `MX ${intents[0].name} is missing (${query.reason})` }; } return evaluateMailMxValues(intents, query.values, previousValues); } catch (error: unknown) { return { valid: false, outcome: 'lookup-error', reason: `MX lookup failed: ${(error as Error).message}` }; } } public async verifyForwardConfirmedIdentity( identity: IEmailDomainEdgeIdentity, ): Promise { const checks: Array<{ family: 'A' | 'AAAA'; address: string }> = []; if (identity.ipv4) checks.push({ family: 'A', address: identity.ipv4 }); if (identity.ipv6) checks.push({ family: 'AAAA', address: identity.ipv6 }); try { for (const check of checks) { const reverseDnsName = plugins.smartnetwork.toReverseDnsName(check.address); if (!reverseDnsName) { return { valid: false, outcome: 'invalid', reason: `Invalid source-bound IP address: ${check.address}`, }; } const ptrQuery = await this.queryValues(reverseDnsName, 'PTR'); if (ptrQuery.status === 'error') { return { valid: false, outcome: 'lookup-error', reason: `PTR lookup failed: ${ptrQuery.reason}` }; } const ptrNames = ptrQuery.status === 'missing' ? [] : ptrQuery.values.map(normalizeHostname); if (ptrNames.length !== 1 || ptrNames[0] !== normalizeHostname(identity.hostname)) { return { valid: false, reason: `PTR for ${check.address} must be exactly ${identity.hostname}`, }; } const forwardQuery = await this.queryValues(identity.hostname, check.family); if (forwardQuery.status === 'error') { return { valid: false, outcome: 'lookup-error', reason: `${check.family} lookup failed: ${forwardQuery.reason}` }; } const forwardIps = forwardQuery.status === 'missing' ? [] : forwardQuery.values; if (!forwardIps.includes(check.address)) { return { valid: false, reason: `${check.family} ${identity.hostname} does not resolve back to ${check.address}`, }; } } return checks.length > 0 ? { valid: true } : { valid: false, reason: `No source-bound address is available for ${identity.edgeId}` }; } catch (error: unknown) { return { valid: false, reason: `FCrDNS lookup failed: ${(error as Error).message}` }; } } public destroy(): void { this.client?.destroy(); this.client = undefined; } } interface IPlannedDomain { doc: EmailDomainDoc; revision: IEmailDomainDnsRevision; intents: IEmailDnsRecordIntent[]; preliminaryErrors: IEmailDomainOperationError[]; egressIdentitySupported: boolean; selectorCorrectSigningSupported: boolean; } interface IZoneReconciliationResult { errors: Map; provenProviderZoneIds: Set; readyZoneIds: Set; } /** * Sole owner of managed mail DNS. API, topology, timer, and startup events only * mark this globally serialized reconciler dirty. */ export class MailDnsSync { private started = false; private stopped = false; private dirty = false; private runPromise?: Promise; private retryTimer?: ReturnType; private schedulePromises = new Set>(); private scheduleGeneration = 0; private lastScheduledReconciliationAt = 0; private requestedGeneration = 0; private completedGeneration = 0; private forceAll = false; private forcedDomainIds = new Set(); private edgeEligibility: MailEdgeEligibility; private edgeEligibilitySignature?: string; private edgeEligibilityCheckTimer?: ReturnType; private edgeEligibilityCheckPromise?: Promise; private pendingEdgeEligibilityReason?: string; private lastEdgeEligibilitySyncAt = 0; private edgeEligibilityDebounceMs = MAIL_DNS_EDGE_ELIGIBILITY_DEBOUNCE_MS; private edgeEligibilityCooldownMs = MAIL_DNS_EDGE_ELIGIBILITY_COOLDOWN_MS; constructor( private dcRouterRef: DcRouter, edgeEligibility?: MailEdgeEligibility, private verifier: IMailDnsVerifier = new SmartDnsMailVerifier(), ) { this.edgeEligibility = edgeEligibility || dcRouterRef.mailEdgeEligibility || new MailEdgeEligibility(dcRouterRef, new ReleasedRemoteIngressEgressIdentitySource(dcRouterRef)); } public async start(): Promise { if (this.started) { await this.runPromise; return; } this.started = true; this.stopped = false; this.edgeEligibilitySignature = undefined; this.pendingEdgeEligibilityReason = undefined; this.lastEdgeEligibilitySyncAt = 0; this.queueRequest(undefined, true); try { await this.kick('startup'); } catch (error: unknown) { this.stopped = true; this.started = false; this.dirty = false; this.scheduleGeneration++; if (this.retryTimer) clearTimeout(this.retryTimer); this.retryTimer = undefined; if (this.edgeEligibilityCheckTimer) clearTimeout(this.edgeEligibilityCheckTimer); this.edgeEligibilityCheckTimer = undefined; this.pendingEdgeEligibilityReason = undefined; await this.edgeEligibilityCheckPromise?.catch(() => undefined); await this.runPromise?.catch(() => undefined); await Promise.all([...this.schedulePromises]); this.verifier.destroy?.(); throw error; } } public async stop(): Promise { this.stopped = true; this.started = false; this.dirty = false; this.scheduleGeneration++; if (this.retryTimer) { clearTimeout(this.retryTimer); this.retryTimer = undefined; } if (this.edgeEligibilityCheckTimer) { clearTimeout(this.edgeEligibilityCheckTimer); this.edgeEligibilityCheckTimer = undefined; } this.pendingEdgeEligibilityReason = undefined; await this.edgeEligibilityCheckPromise?.catch(() => undefined); await this.runPromise?.catch(() => undefined); await Promise.all([...this.schedulePromises]); this.verifier.destroy?.(); } public requestSync(reason: string, domainIds?: string[]): void { if (this.stopped) return; this.queueRequest(domainIds, domainIds === undefined); if (!this.started) return; this.kick(reason).catch((error: unknown) => { logger.log('error', `MailDnsSync failed (${reason}): ${(error as Error).message}`); }); } /** * RemoteIngress topology events are advisory. Resolve the actual mail-edge * eligibility snapshot and only reconcile DNS when its semantic shape * changes. Volatile heartbeat/observation timestamps never enter the * signature. */ public requestEdgeEligibilityCheck(reason: string): void { if (this.stopped || !this.started) return; this.pendingEdgeEligibilityReason = reason; if (this.edgeEligibilityCheckTimer || this.edgeEligibilityCheckPromise) return; this.scheduleEdgeEligibilityCheck(); } private scheduleEdgeEligibilityCheck(): void { if (this.stopped || !this.started || !this.pendingEdgeEligibilityReason) return; const cooldownRemaining = Math.max( 0, this.lastEdgeEligibilitySyncAt + this.edgeEligibilityCooldownMs - Date.now(), ); const delay = Math.max(this.edgeEligibilityDebounceMs, cooldownRemaining); const timer = setTimeout(() => { this.edgeEligibilityCheckTimer = undefined; this.runEdgeEligibilityCheck().catch((error: unknown) => { logger.log('error', `MailDnsSync edge eligibility check failed: ${(error as Error).message}`); }); }, delay) as ReturnType & { unref?: () => void }; this.edgeEligibilityCheckTimer = timer; timer.unref?.(); } private async runEdgeEligibilityCheck(): Promise { if (this.edgeEligibilityCheckPromise) return await this.edgeEligibilityCheckPromise; const reason = this.pendingEdgeEligibilityReason; this.pendingEdgeEligibilityReason = undefined; if (!reason || this.stopped || !this.started) return; this.edgeEligibilityCheckPromise = (async () => { const result = await this.edgeEligibility.resolveForDomain(); const nextSignature = mailEdgeEligibilitySignature(result); const previousSignature = this.edgeEligibilitySignature; this.edgeEligibilitySignature = nextSignature; if ( previousSignature === undefined || previousSignature === nextSignature || this.stopped || !this.started ) { return; } this.lastEdgeEligibilitySyncAt = Date.now(); this.requestSync(reason); })().finally(() => { this.edgeEligibilityCheckPromise = undefined; if (this.pendingEdgeEligibilityReason && !this.stopped && this.started) { this.scheduleEdgeEligibilityCheck(); } }); return await this.edgeEligibilityCheckPromise; } /** Coalescing entry point used by API actions that need the durable outcome. */ public async sync(reason: string, domainIds?: string[]): Promise { if (this.stopped) throw new Error('MailDnsSync is stopped'); const ticket = this.queueRequest(domainIds, domainIds === undefined); while (this.completedGeneration < ticket) { if (this.stopped) throw new Error('MailDnsSync is stopped'); await this.kick(reason); } if (this.stopped) throw new Error('MailDnsSync is stopped'); } public async syncDomain(domainId: string, reason: string): Promise<{ requestedGeneration: number; completedGeneration: number; domain: EmailDomainDoc | null; records: DnsRecordDoc[]; }> { if (this.stopped) throw new Error('MailDnsSync is stopped'); const requestedGeneration = this.queueRequest([domainId], false); while (this.completedGeneration < requestedGeneration) { if (this.stopped) throw new Error('MailDnsSync is stopped'); await this.kick(reason); } if (this.stopped) throw new Error('MailDnsSync is stopped'); const domain = await EmailDomainDoc.findById(domainId); return { requestedGeneration, completedGeneration: this.completedGeneration, domain, records: domain ? await DnsRecordDoc.findByDomainId(domain.linkedDomainId) : [], }; } private queueRequest(domainIds: string[] | undefined, forceAll: boolean): number { this.requestedGeneration++; this.dirty = true; if (forceAll) this.forceAll = true; for (const domainId of domainIds || []) this.forcedDomainIds.add(domainId); return this.requestedGeneration; } private async kick(reason: string): Promise { if (this.stopped) throw new Error('MailDnsSync is stopped'); if (this.runPromise) return await this.runPromise; // Defer execution one microtask so runPromise is assigned before the first // awaited provider/eligibility hook can synchronously enqueue another event. this.runPromise = Promise.resolve().then(() => this.runLoop(reason)).finally(() => { this.runPromise = undefined; if (this.dirty && this.started && !this.stopped) { this.kick(`${reason} (follow-up)`).catch((error: unknown) => { logger.log('error', `MailDnsSync follow-up failed: ${(error as Error).message}`); }); } this.queueScheduleNextDue(); }); return await this.runPromise; } private async runLoop(reason: string): Promise { do { const generation = this.requestedGeneration; const forceAll = this.forceAll; const forcedDomainIds = new Set(this.forcedDomainIds); this.dirty = false; this.forceAll = false; this.forcedDomainIds.clear(); await this.reconcile(reason, { forceAll, forcedDomainIds }); this.completedGeneration = Math.max(this.completedGeneration, generation); // An event during the run leaves dirty=true and forces exactly one // immediate follow-up; repeated events coalesce into that pass. } while (this.dirty && !this.stopped); } private async reconcile( reason: string, work: { forceAll: boolean; forcedDomainIds: Set }, ): Promise { if (this.dcRouterRef.options.dbConfig?.enabled === false) return; const dnsManager = this.dcRouterRef.dnsManager; if (!dnsManager) { logger.log('warn', `MailDnsSync (${reason}): DnsManager unavailable; retry intent retained`); return; } await dnsManager.runManagedMailDnsMutationExclusive( async () => await this.reconcileExclusive(reason, work), ); } private async reconcileExclusive( reason: string, work: { forceAll: boolean; forcedDomainIds: Set }, ): Promise { const zones = await DomainDoc.findAll(); const allDocs = await EmailDomainDoc.findAll(); const now = Date.now(); const docs = allDocs.filter((doc) => ( work.forceAll || work.forcedDomainIds.has(doc.id) || this.isDomainDue(doc, now) )); if (docs.length === 0) return; // Refresh the broad operational status independently from per-domain pins. const broadEligibility = await this.edgeEligibility.resolveForDomain(); this.edgeEligibilitySignature = mailEdgeEligibilitySignature(broadEligibility); const plans: IPlannedDomain[] = []; const deletingDocs: EmailDomainDoc[] = []; for (const loadedDoc of docs) { if (loadedDoc.reconciliation?.lifecycleStatus === 'deleting') { await this.persistDeletingAttempt(loadedDoc); deletingDocs.push(loadedDoc); continue; } const repairedDoc = await this.dcRouterRef.emailDomainManager?.repairManagedDkimIfDue?.(loadedDoc.id) || loadedDoc; if (repairedDoc.reconciliation?.errors.some((error) => error.code === 'DKIM_KEY_REPAIR_FAILED')) { // Key repair owns this retry schedule. DNS planning here would publish // stale DKIM material and overwrite the bounded repair backoff. continue; } const doc = await this.dcRouterRef.emailDomainManager?.stageDkimRotationIfDue?.(repairedDoc.id) || repairedDoc; if (doc.reconciliation?.lifecycleStatus === 'deleting') { await this.persistDeletingAttempt(doc); deletingDocs.push(doc); continue; } const plan = await this.buildPlan(doc, zones); await this.persistDesiredPlan(plan, zones); plans.push(plan); } const uniqueIntents = new Map(); for (const plan of plans) { for (const intent of plan.intents) { uniqueIntents.set(intent.key, intent); } } const zoneResult = await this.reconcileZones([...uniqueIntents.values()], zones); for (const plan of plans) { const errors = [...plan.preliminaryErrors]; for (const intent of plan.intents) { const outcome = uniqueIntents.get(intent.key); if (outcome) Object.assign(intent, outcome); if (intent.error) errors.push(intent.error); } await this.validateAndActivate(plan, errors, zoneResult, zones); } const cleanupProof = await this.refreshProviderZonesForCleanup( [...plans.map((plan) => plan.doc), ...deletingDocs], zones, zoneResult.provenProviderZoneIds, ); await this.cleanupStaleManagedRecords( zones, cleanupProof, new Set(plans.map((plan) => `domain:${plan.doc.id}`).concat( deletingDocs.map((doc) => `domain:${doc.id}`), )), work.forceAll || deletingDocs.length > 0, ); await this.pruneRetiredDkimMaterial(plans.map((plan) => plan.doc)); await this.finalizeDeletingDomains(deletingDocs, zones); await this.dcRouterRef.emailDomainManager?.syncManagedDomainsToRuntime(); await this.dcRouterRef.workAppMailManager?.applyStoredIdentitiesToRuntime(); this.dcRouterRef.opsServer?.invalidateRealtime?.(['emailDomains', 'dns'], { changedIds: docs.map((doc) => doc.id), reason: `mail-dns-reconciled:${reason}`, }); } private isDomainDue(doc: EmailDomainDoc, now: number): boolean { const retryAt = Date.parse(doc.reconciliation?.retryAt || ''); if (Number.isFinite(retryAt) && retryAt <= now) return true; if (Number.isFinite(retryAt)) return false; if (!doc.reconciliation?.lastAttemptAt) return true; const lastAttemptAt = Date.parse(doc.reconciliation.lastAttemptAt); if ( Number.isFinite(lastAttemptAt) && lastAttemptAt + MAIL_DNS_RECONCILIATION_INTERVAL_MS <= now ) return true; const rotationDueAt = this.getDkimRotationDueAt(doc, now); if (rotationDueAt !== undefined && rotationDueAt <= now) return true; const retirementDueAt = this.getDkimRetirementDueAt(doc); return retirementDueAt !== undefined && retirementDueAt <= now; } private getDkimRotationDueAt(doc: EmailDomainDoc, now: number): number | undefined { if (!doc.dkim.rotateKeys || !doc.dkim.rotationIntervalDays) return undefined; const activeDkim = this.getActiveDkim(doc); const lastRotation = Date.parse( activeDkim?.promotedAt || doc.dkim.lastRotatedAt || activeDkim?.createdAt || doc.createdAt, ); if (!Number.isFinite(lastRotation)) return undefined; const dueAt = lastRotation + doc.dkim.rotationIntervalDays * 86_400_000; const canStage = !doc.pendingDkim && this.dcRouterRef.emailDomainManager?.supportsSelectorCorrectSigning?.() === true && typeof this.dcRouterRef.emailDomainManager?.stageDkimRotationIfDue === 'function'; if (canStage) return dueAt; const lastAttemptAt = Date.parse(doc.reconciliation?.lastAttemptAt || ''); const retryReference = Number.isFinite(lastAttemptAt) ? lastAttemptAt : now; return Math.max(dueAt, retryReference + MAIL_DNS_RETRY_INTERVAL_MS); } private getDkimRetirementDueAt(doc: EmailDomainDoc): number | undefined { const dueTimes = (doc.retiringDkim || []) .map((material) => Date.parse(material.retireAfter || '')) .filter(Number.isFinite); return dueTimes.length > 0 ? Math.min(...dueTimes) : undefined; } private async scheduleNextDue(): Promise { const generation = ++this.scheduleGeneration; if (this.retryTimer) clearTimeout(this.retryTimer); this.retryTimer = undefined; if (!this.started || this.stopped) return; const dueTimes: number[] = []; const now = Date.now(); const docs = await EmailDomainDoc.findAll(); if (generation !== this.scheduleGeneration || !this.started || this.stopped) return; for (const doc of docs) { const retryAt = Date.parse(doc.reconciliation?.retryAt || ''); if (Number.isFinite(retryAt)) { dueTimes.push(retryAt); continue; } const lastAttemptAt = Date.parse(doc.reconciliation?.lastAttemptAt || ''); dueTimes.push(Number.isFinite(lastAttemptAt) ? lastAttemptAt + MAIL_DNS_RECONCILIATION_INTERVAL_MS : now); const rotationDueAt = this.getDkimRotationDueAt(doc, now); if (rotationDueAt !== undefined) dueTimes.push(rotationDueAt); const retirementDueAt = this.getDkimRetirementDueAt(doc); if (retirementDueAt !== undefined) dueTimes.push(retirementDueAt); } if (dueTimes.length === 0) return; const nextDueAt = Math.max( Math.min(...dueTimes), this.lastScheduledReconciliationAt + MAIL_DNS_RECONCILIATION_INTERVAL_MS, ); const delay = Math.max(1, Math.min(2_147_483_647, nextDueAt - now)); this.retryTimer = setTimeout(() => { this.retryTimer = undefined; if (this.stopped || !this.started) return; this.lastScheduledReconciliationAt = Date.now(); this.queueRequest(undefined, true); this.kick('scheduled desired-state reconciliation').catch((error: unknown) => { logger.log('error', `MailDnsSync scheduled retry failed: ${(error as Error).message}`); }); }, delay); } private queueScheduleNextDue(): void { const schedulePromise = this.scheduleNextDue() .catch((error: unknown) => { logger.log('error', `MailDnsSync scheduling failed: ${(error as Error).message}`); }) .finally(() => { this.schedulePromises.delete(schedulePromise); }); this.schedulePromises.add(schedulePromise); } private async buildPlan(doc: EmailDomainDoc, zones: DomainDoc[]): Promise { const now = new Date().toISOString(); const eligibility = await this.edgeEligibility.resolveForDomain(doc); const preliminaryErrors: IEmailDomainOperationError[] = [ ...(doc.reconciliation?.errors || []).filter((error) => ( error.code === 'DKIM_ROTATION_STAGE_FAILED' || error.code === 'DKIM_KEY_REPAIR_FAILED' )), ]; if (!eligibility.supported) { preliminaryErrors.push({ code: 'EGRESS_IDENTITY_PROOF_UNAVAILABLE', message: eligibility.reason || 'RemoteIngress egress identity proof is unavailable', retryable: true, }); } else if (eligibility.edges.length === 0) { preliminaryErrors.push({ code: 'NO_ELIGIBLE_MAIL_EDGE', message: eligibility.reason || 'No eligible mail edge is connected', retryable: true, }); } else if (eligibility.edges.length < 2) { preliminaryErrors.push({ code: 'INSUFFICIENT_ELIGIBLE_MAIL_EDGES', message: 'Two eligible RemoteIngress mail edges are required; direct hub fallback is disabled', retryable: true, }); } const activeDkim = this.getActiveDkim(doc); const upstreamSelectorCapability = this.dcRouterRef.emailDomainManager?.supportsSelectorCorrectSigning?.() === true; const selectorCorrectSigningSupported = ( (!activeDkim || activeDkim.selector === 'default') && !doc.pendingDkim ) || upstreamSelectorCapability; if (!activeDkim) { preliminaryErrors.push({ code: 'DKIM_KEY_UNAVAILABLE', message: `No usable active DKIM key exists for ${doc.domain}`, retryable: false, }); } else if (!selectorCorrectSigningSupported) { preliminaryErrors.push({ code: 'SMARTMTA_SELECTOR_SIGNING_UNSUPPORTED', message: 'The consumed SmartMTA release cannot safely sign with a non-default selector', retryable: false, }); } const mxSlots = this.allocateMxSlots(doc, eligibility.edges); if (eligibility.edges.length >= 2 && mxSlots.length < 2) { preliminaryErrors.push({ code: 'INSUFFICIENT_DISTINCT_MX_IDENTITIES', message: 'At least two eligible mail edges exist but they do not provide two distinct MX hostnames', retryable: true, }); } const candidatesById = new Map( eligibility.edges.map((candidate) => [candidate.edge.id, candidate]), ); const selectedEdges: IEligibleMailEdge[] = mxSlots.flatMap((slot) => { const candidate = candidatesById.get(slot.edgeId); return candidate ? [candidate] : []; }); const edgeIdentities = selectedEdges.map((candidate) => this.toDomainEdgeIdentity(candidate)); const revisionId = plugins.smartunique.shortId(); const intents: IEmailDnsRecordIntent[] = []; for (const candidate of selectedEdges) { for (const identity of candidate.identities) { const type = identity.family === 4 ? 'A' : 'AAAA'; intents.push(this.intentForName(zones, { key: `mail-topology:${candidate.edge.id}:${type}:${candidate.hostname}:${identity.address}`, ownerType: 'topology', ownerId: `topology:${candidate.edge.id}`, name: candidate.hostname, type, value: identity.address, ttl: 300, })); } } if (mxSlots.length === 2) { for (const slot of mxSlots) { intents.push(this.intentForName(zones, { key: `mail-domain:${doc.id}:mx:${slot.priority}:${slot.edgeId}:${slot.hostname}`, ownerType: 'domain', ownerId: `domain:${doc.id}`, name: doc.domain, type: 'MX', value: `${slot.priority} ${slot.hostname}`, ttl: 300, })); } const spfIdentities = [ ...edgeIdentities, ...(doc.reconciliation?.activeRevision?.edgeIdentities || []), ]; intents.push(this.intentForName(zones, { key: `mail-domain:${doc.id}:spf`, ownerType: 'domain', ownerId: `domain:${doc.id}`, name: doc.domain, type: 'TXT', value: this.buildSpfValue(spfIdentities), ttl: 300, })); } if (activeDkim) intents.push(this.buildDkimIntent(doc, activeDkim, zones)); if (doc.pendingDkim) intents.push(this.buildDkimIntent(doc, doc.pendingDkim, zones)); for (const retiring of doc.retiringDkim || []) { if (!retiring.retireAfter || Date.parse(retiring.retireAfter) > Date.now()) { intents.push(this.buildDkimIntent(doc, retiring, zones)); } } intents.push(this.intentForName(zones, { key: `mail-domain:${doc.id}:dmarc`, ownerType: 'domain', ownerId: `domain:${doc.id}`, name: `_dmarc.${doc.domain}`, type: 'TXT', value: `v=DMARC1; p=none; rua=mailto:dmarc@${doc.domain}`, ttl: 3600, })); for (const intent of intents) { if (!intent.domainId) { intent.status = 'failed'; intent.error = { code: 'DNS_ZONE_UNMANAGED', message: `No managed DNS zone covers ${intent.name}`, recordKey: intent.key, retryable: false, }; } } const revision: IEmailDomainDnsRevision = { id: revisionId, createdAt: now, selector: doc.pendingDkim?.selector || activeDkim?.selector || doc.dkim.selector || 'default', edgeIdentities, mxSlots, recordKeys: intents.map((intent) => intent.key), }; return { doc, revision, intents, preliminaryErrors, egressIdentitySupported: eligibility.supported, selectorCorrectSigningSupported, }; } private getActiveDkim(doc: EmailDomainDoc): IEmailDkimMaterial | undefined { if (doc.activeDkim?.publicKey) return doc.activeDkim; const publicKey = doc.dkim?.publicKey?.trim(); if (!publicKey) return undefined; return { selector: doc.dkim.selector || 'default', keySize: doc.dkim.keySize || 2048, publicKey, createdAt: doc.createdAt, }; } private buildDkimIntent( doc: EmailDomainDoc, material: IEmailDkimMaterial, zones: DomainDoc[], ): IEmailDnsRecordIntent { return this.intentForName(zones, { key: `mail-domain:${doc.id}:dkim:${material.selector}`, ownerType: 'domain', ownerId: `domain:${doc.id}`, name: `${material.selector}._domainkey.${doc.domain}`, type: 'TXT', value: `v=DKIM1; h=sha256; k=rsa; p=${material.publicKey}`, ttl: 3600, }); } private intentForName( zones: DomainDoc[], input: Omit, ): IEmailDnsRecordIntent { return { ...input, domainId: this.findZoneForName(input.name, zones)?.id || '', status: 'pending', }; } private toDomainEdgeIdentity(candidate: IEligibleMailEdge): IEmailDomainEdgeIdentity { const ipv4 = candidate.identities.find((identity) => identity.family === 4); const ipv6 = candidate.identities.find((identity) => identity.family === 6); const observedAt = Math.min(...candidate.identities.map((identity) => identity.observedAt)); const hubReceivedAt = Math.min(...candidate.identities.map((identity) => identity.hubReceivedAt)); return { edgeId: candidate.edge.id, hostname: candidate.hostname, ...(ipv4 ? { ipv4: ipv4.address } : {}), ...(ipv6 ? { ipv6: ipv6.address } : {}), observedAt: new Date(observedAt).toISOString(), hubReceivedAt: new Date(hubReceivedAt).toISOString(), }; } private allocateMxSlots(doc: EmailDomainDoc, candidates: IEligibleMailEdge[]): IEmailDomainMxSlot[] { if (candidates.length < 2) return []; const byId = new Map(candidates.map((candidate) => [candidate.edge.id, candidate])); const result: IEmailDomainMxSlot[] = []; const usedEdges = new Set(); const usedHostnames = new Set(); const previous = [ ...(doc.reconciliation?.activeRevision?.mxSlots || []), ...(doc.reconciliation?.pendingRevision?.mxSlots || []), ]; for (const priority of [10, 20] as const) { const old = previous.find((slot) => slot.priority === priority); const candidate = old ? byId.get(old.edgeId) : undefined; if (!candidate || candidate.hostname !== old?.hostname) continue; if (usedEdges.has(candidate.edge.id) || usedHostnames.has(candidate.hostname)) continue; result.push({ priority, edgeId: candidate.edge.id, hostname: candidate.hostname }); usedEdges.add(candidate.edge.id); usedHostnames.add(candidate.hostname); } for (const priority of [10, 20] as const) { if (result.some((slot) => slot.priority === priority)) continue; const candidate = candidates.find((entry) => ( !usedEdges.has(entry.edge.id) && !usedHostnames.has(entry.hostname) )); if (!candidate) continue; result.push({ priority, edgeId: candidate.edge.id, hostname: candidate.hostname }); usedEdges.add(candidate.edge.id); usedHostnames.add(candidate.hostname); } return result.length === 2 ? result.sort((left, right) => left.priority - right.priority) : []; } private buildSpfValue(identities: IEmailDomainEdgeIdentity[]): string { const terms = new Set(); for (const identity of identities) { if (identity.ipv4) terms.add(`ip4:${identity.ipv4}`); if (identity.ipv6) terms.add(`ip6:${identity.ipv6}`); } return `v=spf1 ${[...terms].sort().join(' ')} -all`.replace(/\s+/g, ' ').trim(); } private async persistDesiredPlan(plan: IPlannedDomain, zones: DomainDoc[]): Promise { const previous = plan.doc.reconciliation; const previousSignature = JSON.stringify((previous?.intents || []).map((intent) => [intent.key, intent.value])); const nextSignature = JSON.stringify(plan.intents.map((intent) => [intent.key, intent.value])); const generation = (previous?.desiredGeneration || 0) + (previousSignature === nextSignature ? 0 : 1); const desiredGeneration = Math.max(generation, 1); plan.revision.generation = desiredGeneration; const providerZoneIds = new Set(); const providerZones = new Set(zones.filter((zone) => zone.source === 'provider').map((zone) => zone.id)); for (const intent of plan.intents) { if (providerZones.has(intent.domainId)) providerZoneIds.add(intent.domainId); } const linkedZone = zones.find((zone) => zone.id === plan.doc.linkedDomainId); if (linkedZone?.source === 'provider') providerZoneIds.add(linkedZone.id); for (const record of await DnsRecordDoc.findAll()) { if ( record.managedBy === MAIL_DNS_MANAGED_BY && record.managedOwnerId === `domain:${plan.doc.id}` && providerZones.has(record.domainId) ) providerZoneIds.add(record.domainId); } plan.doc.reconciliation = { lifecycleStatus: previous?.lifecycleStatus === 'active' ? 'active' : 'pending', desiredGeneration, activeRevision: previous?.activeRevision, pendingRevision: plan.revision, intents: plan.intents, errors: plan.preliminaryErrors, capability: { egressIdentityProof: plan.egressIdentitySupported ? 'supported' : 'unsupported', selectorCorrectSigning: plan.selectorCorrectSigningSupported ? 'supported' : 'unsupported', }, providerZoneIds: [...providerZoneIds].sort(), retryAttempt: previous?.retryAttempt, lastAttemptAt: new Date().toISOString(), retryAt: new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString(), }; plan.doc.updatedAt = new Date().toISOString(); await plan.doc.save(); } private async persistDeletingAttempt(doc: EmailDomainDoc): Promise { const deletingProviderRecordIds = new Set(doc.reconciliation?.deletingProviderRecordIds || []); const deletingRecordKeys = new Set([ ...(doc.reconciliation?.activeRevision?.recordKeys || []), ...(doc.reconciliation?.pendingRevision?.recordKeys || []), ]); const retainedTopologyKeys = new Set(); for (const otherDoc of await EmailDomainDoc.findAll()) { if (otherDoc.id === doc.id || otherDoc.reconciliation?.lifecycleStatus === 'deleting') continue; for (const intent of otherDoc.reconciliation?.intents || []) retainedTopologyKeys.add(intent.key); for (const key of otherDoc.reconciliation?.activeRevision?.recordKeys || []) retainedTopologyKeys.add(key); for (const key of otherDoc.reconciliation?.pendingRevision?.recordKeys || []) retainedTopologyKeys.add(key); } const hasKnownProviderProvenance = doc.reconciliation?.providerZoneIds !== undefined; const providerZoneIds = new Set(doc.reconciliation?.providerZoneIds || []); for (const record of await DnsRecordDoc.findAll()) { const ownedByDomain = record.managedOwnerId === `domain:${doc.id}`; const unsharedDeletingTopology = record.managedOwnerId?.startsWith('topology:') && Boolean(record.managedRecordKey && deletingRecordKeys.has(record.managedRecordKey)) && Boolean(record.managedRecordKey && !retainedTopologyKeys.has(record.managedRecordKey)); if ( record.managedBy === MAIL_DNS_MANAGED_BY && (ownedByDomain || unsharedDeletingTopology) && record.providerRecordId?.trim() ) { deletingProviderRecordIds.add(record.providerRecordId.trim()); if (hasKnownProviderProvenance) providerZoneIds.add(record.domainId); } } doc.reconciliation = { ...(doc.reconciliation || { lifecycleStatus: 'deleting', desiredGeneration: 1, intents: [], errors: [], }), lifecycleStatus: 'deleting', intents: [], errors: (doc.reconciliation?.errors || []).filter((error) => error.code !== 'DNS_CLEANUP_FAILED'), lastAttemptAt: new Date().toISOString(), retryAt: new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString(), ...(hasKnownProviderProvenance ? { providerZoneIds: [...providerZoneIds].sort() } : {}), deletingProviderRecordIds: [...deletingProviderRecordIds].sort(), }; await doc.save(); } private async reconcileZones( intents: IEmailDnsRecordIntent[], zones: DomainDoc[], ): Promise { const errors = new Map(); const provenProviderZoneIds = new Set(); const readyZoneIds = new Set(); const byZone = new Map(); for (const intent of intents) { if (!intent.domainId || intent.error) continue; const group = byZone.get(intent.domainId) || []; group.push(intent); byZone.set(intent.domainId, group); } for (const [zoneId, zoneIntents] of byZone) { const zone = zones.find((candidate) => candidate.id === zoneId); if (!zone) continue; try { if (zone.source === 'provider') { const result = await this.dcRouterRef.dnsManager!.syncDomain(zone.id); if (!result.success) throw new Error(result.message || `Failed to refresh ${zone.name}`); provenProviderZoneIds.add(zone.id); readyZoneIds.add(zone.id); } if ( zone.source === 'dcrouter' && this.dcRouterRef.dnsServer && this.dcRouterRef.dnsManager?.dnsServer === this.dcRouterRef.dnsServer ) { readyZoneIds.add(zone.id); } for (const intent of this.sortIntents(zoneIntents)) { await this.reconcileIntent(intent, zoneIntents); } } catch (error: unknown) { const outcome: IEmailDomainOperationError = { code: 'DNS_PROVIDER_ZONE_FAILED', message: `DNS zone ${zone.name} reconciliation failed: ${(error as Error).message}`, retryable: true, }; errors.set(zone.id, outcome); for (const intent of zoneIntents) { intent.status = 'pending'; intent.verificationOutcome = 'lookup-error'; intent.error = { ...outcome, recordKey: intent.key }; } } } return { errors, provenProviderZoneIds, readyZoneIds }; } private sortIntents(intents: IEmailDnsRecordIntent[]): IEmailDnsRecordIntent[] { const rank = (intent: IEmailDnsRecordIntent): number => { if (intent.type === 'A' || intent.type === 'AAAA') return 0; if (intent.type === 'TXT' && intent.name.includes('._domainkey.')) return 1; if (intent.type === 'TXT') return 2; return 3; // MX is exposed only after identity/auth records exist. }; return [...intents].sort((left, right) => rank(left) - rank(right) || left.key.localeCompare(right.key)); } private async reconcileIntent( intent: IEmailDnsRecordIntent, zoneIntents: IEmailDnsRecordIntent[], ): Promise { const records = await DnsRecordDoc.findByDomainId(intent.domainId); const rrset = records.filter((record) => ( record.name.toLowerCase() === intent.name.toLowerCase() && record.type === intent.type )); const ownedByKey = records.filter((record) => ( record.managedBy === MAIL_DNS_MANAGED_BY && record.managedOwnerId === intent.ownerId && record.managedRecordKey === intent.key )); if (ownedByKey.length > 1) { intent.status = 'conflict'; intent.verificationOutcome = 'duplicate'; intent.error = { code: 'DNS_MANAGED_KEY_DUPLICATE', message: `Multiple managed DNS records share ownership key ${intent.key}`, recordKey: intent.key, retryable: false, }; return; } const reconciliationRecords = [...new Map( [...rrset, ...ownedByKey].map((record) => [record.id, record]), ).values()]; if (await reconcileMailDnsSingleton({ intent, rrset: reconciliationRecords, dnsManager: this.dcRouterRef.dnsManager!, managedBy: MAIL_DNS_MANAGED_BY, createdBy: MAIL_DNS_SYNC_CREATED_BY, evaluate: async (candidate, values) => await this.evaluateTxtRecords(candidate, values), })) return; const exactOwner = reconciliationRecords .filter((record) => ( record.managedBy === MAIL_DNS_MANAGED_BY && record.managedOwnerId === intent.ownerId && record.managedRecordKey === intent.key )) .sort((left, right) => left.id.localeCompare(right.id)); const exact = rrset.filter((record) => this.valuesEqual(intent.type, record.value, intent.value)); const exactOtherManaged = exact.filter((record) => ( record.managedBy === MAIL_DNS_MANAGED_BY && (record.managedOwnerId !== intent.ownerId || record.managedRecordKey !== intent.key) )); if (exactOtherManaged.length > 0) { intent.status = 'conflict'; intent.verificationOutcome = 'conflict'; intent.error = { code: 'DNS_MANAGED_OWNER_CONFLICT', message: `Another managed owner controls ${intent.type} ${intent.name}`, recordKey: intent.key, retryable: false, }; return; } const desiredValues = zoneIntents .filter((candidate) => candidate.name.toLowerCase() === intent.name.toLowerCase() && candidate.type === intent.type) .map((candidate) => candidate.value); const unmanaged = rrset.filter((record) => ( record.managedBy !== MAIL_DNS_MANAGED_BY && this.isRelevantUnmanagedTxt(intent, record.value) )); const exactUnmanaged = exact.filter((record) => record.managedBy !== MAIL_DNS_MANAGED_BY); const conflictingUnmanaged = unmanaged.filter((record) => ( !desiredValues.some((desired) => this.valuesEqual(intent.type, record.value, desired)) )); if (conflictingUnmanaged.length > 0) { intent.status = 'conflict'; intent.verificationOutcome = 'conflict'; intent.error = { code: 'DNS_UNMANAGED_CONFLICT', message: `Unmanaged ${intent.type} record conflicts with ${intent.name}`, recordKey: intent.key, retryable: false, }; return; } if (exactUnmanaged.length > 0) { // Exact operator record satisfies the intent without adoption/deletion. // Remove only dcrouter-owned duplicates of this same intent; other MX // values in the desired RRset remain independently managed. for (const duplicate of exactOwner) { const result = await this.dcRouterRef.dnsManager!.deleteRecord(duplicate.id); if (!result.success) throw new Error(result.message || `Failed to delete duplicate ${intent.key}`); } if (exactUnmanaged.length > 1) { intent.status = 'conflict'; intent.verificationOutcome = 'duplicate'; intent.error = { code: 'DNS_PUBLIC_RRSET_DUPLICATE', message: `Operator ${intent.type} RRset has duplicate values at ${intent.name}`, recordKey: intent.key, retryable: false, }; return; } intent.status = 'satisfied'; intent.satisfactionSource = 'operator'; return; } const managed = exactOwner.find((record) => this.valuesEqual(intent.type, record.value, intent.value)) || exactOwner[0]; if (managed) { if ( !this.valuesEqual(intent.type, managed.value, intent.value) || managed.name !== intent.name || managed.type !== intent.type || managed.ttl !== intent.ttl ) { const result = await this.dcRouterRef.dnsManager!.updateRecord({ id: managed.id, name: intent.name, type: intent.type, value: intent.value, ttl: intent.ttl, }); if (!result.success) throw new Error(result.message || `Failed to update ${intent.key}`); } for (const duplicate of exactOwner.filter((record) => record.id !== managed.id)) { const result = await this.dcRouterRef.dnsManager!.deleteRecord(duplicate.id); if (!result.success) throw new Error(result.message || `Failed to delete duplicate ${intent.key}`); } intent.providerRecordId = managed.providerRecordId; intent.status = 'satisfied'; intent.satisfactionSource = 'managed'; intent.lastAttemptAt = new Date().toISOString(); return; } const result = await this.dcRouterRef.dnsManager!.createRecord({ domainId: intent.domainId, name: intent.name, type: intent.type, value: intent.value, ttl: intent.ttl, createdBy: MAIL_DNS_SYNC_CREATED_BY, managedBy: MAIL_DNS_MANAGED_BY, managedOwnerId: intent.ownerId, managedRecordKey: intent.key, }); if (!result.success) throw new Error(result.message || `Failed to create ${intent.key}`); intent.status = 'satisfied'; intent.lastAttemptAt = new Date().toISOString(); } private async evaluateTxtRecords( intent: IEmailDnsRecordIntent, values: string[], ): Promise { return this.verifier.evaluateTxtRecords ? await this.verifier.evaluateTxtRecords(intent, values) : await evaluateMailTxtIntent(intent, values); } private valuesEqual(type: TDnsRecordType, left: string, right: string): boolean { if (type === 'MX') return normalizeMx(left) === normalizeMx(right); if (type === 'A' || type === 'AAAA') return left.trim() === right.trim(); return left.trim() === right.trim(); } private isRelevantUnmanagedTxt(intent: IEmailDnsRecordIntent, value: string): boolean { if (intent.type !== 'TXT') return true; const normalizedIntent = intent.value.trim().toLowerCase(); const normalizedValue = value.trim().toLowerCase(); if (normalizedIntent.startsWith('v=spf1')) return normalizedValue.startsWith('v=spf1'); if (normalizedIntent.startsWith('v=dmarc1')) return normalizedValue.startsWith('v=dmarc1'); if (normalizedIntent.startsWith('v=dkim1')) return normalizedValue.startsWith('v=dkim1'); return this.valuesEqual(intent.type, value, intent.value); } private async validateAndActivate( plan: IPlannedDomain, errors: IEmailDomainOperationError[], zoneResult: IZoneReconciliationResult, zones: DomainDoc[], ): Promise { for (const intent of plan.intents) { if (intent.error || zoneResult.readyZoneIds.has(intent.domainId)) continue; const zone = zones.find((candidate) => candidate.id === intent.domainId); const providerManaged = zone?.source === 'provider'; const outcome: IEmailDomainOperationError = { code: providerManaged ? 'DNS_PROVIDER_ZONE_UNPROVEN' : 'DNS_SERVER_NOT_READY', message: providerManaged ? `DNS provider zone ${zone?.name || intent.domainId} was not proven by this reconciliation pass` : `Authoritative DNS runtime is not ready for ${zone?.name || intent.domainId}`, retryable: true, }; intent.status = 'pending'; intent.verificationOutcome = 'lookup-error'; intent.error = { ...outcome, recordKey: intent.key }; if (!errors.some((error) => error.code === outcome.code)) errors.push(outcome); } const verifiable = plan.intents.filter((intent) => ( !intent.error && !zoneResult.errors.has(intent.domainId) && zoneResult.readyZoneIds.has(intent.domainId) )); const markResult = (intent: IEmailDnsRecordIntent, result: IMailDnsVerificationResult) => { const outcome = result.outcome || (result.valid ? 'valid' : 'propagation'); applyMailDnsVerificationResult(intent, { outcome, reason: result.reason, effectiveValue: result.effectiveValue, }, 'verify'); }; const mxByName = new Map(); for (const intent of verifiable.filter((candidate) => candidate.type === 'MX')) { const group = mxByName.get(intent.name) || []; group.push(intent); mxByName.set(intent.name, group); } await Promise.all([ ...verifiable.filter((intent) => intent.type !== 'MX').map(async (intent) => { const result = await this.verifier.verifyRecord(intent) .catch((error: unknown) => ({ valid: false, outcome: 'lookup-error' as const, reason: (error as Error).message, })); markResult(intent, result); }), ...[...mxByName.values()].map(async (intents) => { const previousMx = (plan.doc.reconciliation?.activeRevision?.mxSlots || []) .map((slot) => `${slot.priority} ${slot.hostname}`); const result = await (this.verifier.verifyMxSet ? this.verifier.verifyMxSet(intents, previousMx) : Promise.all(intents.map((intent) => this.verifier.verifyRecord(intent))).then((results) => { const precedence: TEmailDnsVerificationOutcome[] = [ 'lookup-error', 'invalid', 'conflict', 'duplicate', 'missing', 'propagation', 'valid', ]; const selected = precedence.find((outcome) => results.some((entry) => ( (entry.outcome || (entry.valid ? 'valid' : 'propagation')) === outcome ))) || 'propagation'; return { valid: selected === 'valid', outcome: selected, reason: results.find((entry) => !entry.valid)?.reason, }; })) .catch((error: unknown) => ({ valid: false, outcome: 'lookup-error' as const, reason: (error as Error).message, })); for (const intent of intents) markResult(intent, result); }), ]); for (const intent of plan.intents) { if (intent.error && !errors.some((error) => error.code === intent.error!.code && error.recordKey === intent.key)) { errors.push(intent.error); } } if (plan.egressIdentitySupported && plan.revision.edgeIdentities.length > 0) { const identityResults = await Promise.all(plan.revision.edgeIdentities.map(async (identity) => ({ identity, result: await this.verifier.verifyForwardConfirmedIdentity(identity) .catch((error: unknown) => ({ valid: false, outcome: 'lookup-error' as const, reason: (error as Error).message, })), }))); for (const { identity, result } of identityResults) { if (!result.valid) { errors.push({ code: 'FCRDNS_INVALID', message: result.reason || `FCrDNS validation failed for ${identity.edgeId}`, retryable: true, }); } } } const canActivate = plan.egressIdentitySupported && plan.revision.edgeIdentities.length === 2 && plan.revision.mxSlots.length === 2 && plan.selectorCorrectSigningSupported && Boolean(this.getActiveDkim(plan.doc)) && plan.intents.length > 0 && plan.intents.every((intent) => intent.status === 'valid') && errors.length === 0; const hasPermanentError = errors.some((error) => error.retryable === false); const hasOperationalError = errors.some((error) => ( error.retryable !== false && ( error.code.includes('LOOKUP') || error.code.includes('PROVIDER') || error.code.includes('EGRESS') || error.code.includes('ELIGIBLE') ) )) || plan.intents.some((intent) => intent.verificationOutcome === 'lookup-error'); const previousRetryAttempt = plan.doc.reconciliation?.retryAttempt || 0; const retryAttempt = canActivate || hasPermanentError ? undefined : hasOperationalError ? previousRetryAttempt + 1 : 0; const retryDelay = hasOperationalError ? Math.min( MAIL_DNS_RETRY_INTERVAL_MS * (2 ** Math.max(0, (retryAttempt || 1) - 1)), MAIL_DNS_MAX_RETRY_INTERVAL_MS, ) : MAIL_DNS_RETRY_INTERVAL_MS; const previousActive = plan.doc.reconciliation?.activeRevision; const topologyChanged = Boolean(previousActive) && JSON.stringify({ selector: previousActive?.selector, edgeIdentities: previousActive?.edgeIdentities, mxSlots: previousActive?.mxSlots, }) !== JSON.stringify({ selector: plan.revision.selector, edgeIdentities: plan.revision.edgeIdentities, mxSlots: plan.revision.mxSlots, }); const now = new Date().toISOString(); if (canActivate && plan.doc.pendingDkim) { this.promotePendingDkim(plan.doc, now); } plan.doc.reconciliation = { lifecycleStatus: canActivate ? 'active' : hasPermanentError ? 'failed' : 'pending', desiredGeneration: plan.doc.reconciliation?.desiredGeneration || 1, activeRevision: canActivate ? { ...plan.revision, activatedAt: now } : plan.doc.reconciliation?.activeRevision, pendingRevision: canActivate ? undefined : plan.revision, intents: plan.intents, errors, capability: { egressIdentityProof: plan.egressIdentitySupported ? 'supported' : 'unsupported', selectorCorrectSigning: plan.selectorCorrectSigningSupported ? 'supported' : 'unsupported', }, providerZoneIds: plan.doc.reconciliation?.providerZoneIds || [], retryAttempt, lastAttemptAt: now, ...(canActivate ? { lastSuccessAt: now } : hasPermanentError ? {} : { retryAt: new Date(Date.now() + retryDelay).toISOString() }), }; const mailPublicationReady = plan.egressIdentitySupported && plan.revision.edgeIdentities.length === 2 && plan.revision.mxSlots.length === 2; plan.doc.dnsStatus = { mx: mailPublicationReady ? projectMailDnsIntentAggregateStatus(plan.intents, (intent) => intent.type === 'MX') : 'unchecked', spf: mailPublicationReady ? projectMailDnsIntentAggregateStatus(plan.intents, (intent) => intent.type === 'TXT' && intent.value.startsWith('v=spf1')) : 'unchecked', dkim: projectMailDnsIntentAggregateStatus(plan.intents, (intent) => intent.type === 'TXT' && intent.name.includes('._domainkey.')), dmarc: projectMailDnsIntentAggregateStatus(plan.intents, (intent) => intent.type === 'TXT' && intent.value.startsWith('v=DMARC1')), lastCheckedAt: now, }; plan.doc.updatedAt = now; await plan.doc.save(); if (canActivate && topologyChanged) { // The transition SPF intentionally authorizes old + new identities. Run // one immediate follow-up to contract it to the newly active revision. this.queueRequest([plan.doc.id], false); } } private promotePendingDkim(doc: EmailDomainDoc, now: string): void { const pending = doc.pendingDkim; if (!pending) return; const previous = this.getActiveDkim(doc); const overlapDays = Math.max(1, doc.dkim.retirementOverlapDays || 30); const retiring = (doc.retiringDkim || []).filter((material) => ( material.selector !== pending.selector && (!material.retireAfter || Date.parse(material.retireAfter) > Date.now()) )); if (previous && previous.selector !== pending.selector) { retiring.push({ ...previous, retireAfter: new Date(Date.now() + overlapDays * 86_400_000).toISOString(), }); } doc.activeDkim = { ...pending, validatedAt: now, promotedAt: now }; doc.pendingDkim = undefined; doc.retiringDkim = retiring; doc.dkim.selector = pending.selector; doc.dkim.keySize = pending.keySize; doc.dkim.publicKey = pending.publicKey; doc.dkim.lastRotatedAt = now; } private async refreshProviderZonesForCleanup( docs: EmailDomainDoc[], zones: DomainDoc[], initialProof: Set, ): Promise> { const provenZoneIds = new Set(initialProof); const ownerDocsByZone = new Map>(); for (const doc of docs) { if (!doc.reconciliation) continue; if (doc.reconciliation.providerZoneIds === undefined) { doc.reconciliation.errors.push({ code: 'DNS_PROVIDER_PROVENANCE_UNKNOWN', message: 'Provider-zone provenance is unknown; destructive DNS cleanup is blocked', retryable: true, }); doc.reconciliation.retryAt = new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString(); await doc.save(); continue; } for (const zoneId of doc.reconciliation.providerZoneIds) { if (provenZoneIds.has(zoneId)) continue; const owners = ownerDocsByZone.get(zoneId) || new Set(); owners.add(doc); ownerDocsByZone.set(zoneId, owners); } } for (const [zoneId, owners] of ownerDocsByZone) { const zone = zones.find((candidate) => candidate.id === zoneId); try { if (!zone || zone.source !== 'provider') { throw new Error(`Provider zone ${zoneId} is not available`); } const result = await this.dcRouterRef.dnsManager!.syncDomain(zoneId); if (!result.success) throw new Error(result.message || `Failed to refresh ${zone.name}`); provenZoneIds.add(zoneId); } catch (error: unknown) { for (const owner of owners) { if (!owner.reconciliation) continue; owner.reconciliation.errors.push({ code: 'DNS_CLEANUP_FAILED', message: `Failed to refresh provider zone ${zone?.name || zoneId} before cleanup: ${(error as Error).message}`, retryable: true, }); owner.reconciliation.retryAt = new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString(); await owner.save(); } } } return provenZoneIds; } private async cleanupStaleManagedRecords( zones: DomainDoc[], provenProviderZoneIds: Set, selectedOwnerIds: Set, allowTopologyCleanup: boolean, ): Promise { const docs = await EmailDomainDoc.findAll(); const keep = new Set(); for (const doc of docs) { if (doc.reconciliation?.lifecycleStatus === 'deleting') continue; for (const intent of doc.reconciliation?.intents || []) keep.add(intent.key); const hasCompleteDesiredMxSet = (doc.reconciliation?.intents || []) .filter((intent) => intent.type === 'MX').length === 2; if (doc.reconciliation?.lifecycleStatus !== 'active' && hasCompleteDesiredMxSet) { for (const key of doc.reconciliation?.activeRevision?.recordKeys || []) keep.add(key); } } const managed = (await DnsRecordDoc.findAll()).filter((record) => record.managedBy === MAIL_DNS_MANAGED_BY); const staleRecords = managed.filter((record) => { if (!record.managedOwnerId || !record.managedRecordKey) return false; if ( !selectedOwnerIds.has(record.managedOwnerId) && !(allowTopologyCleanup && record.managedOwnerId.startsWith('topology:')) ) return false; if (keep.has(record.managedRecordKey)) return false; const zone = zones.find((candidate) => candidate.id === record.domainId); return zone?.source !== 'provider' || provenProviderZoneIds.has(zone.id); }); if (staleRecords.length === 0) return; const results = await this.dcRouterRef.dnsManager!.deleteRecords( staleRecords.map((record) => record.id), ); for (let index = 0; index < staleRecords.length; index++) { const record = staleRecords[index]; const result = results[index]; if (result?.success) continue; const error = new Error(result?.message || 'delete failed'); const domainOwnerId = record.managedOwnerId?.startsWith('domain:') ? record.managedOwnerId.slice('domain:'.length) : undefined; const affectedOwners = domainOwnerId ? docs.filter((doc) => doc.id === domainOwnerId) : docs.filter((doc) => selectedOwnerIds.has(`domain:${doc.id}`)); if (affectedOwners.length > 0) { for (const owner of affectedOwners) { if (!owner.reconciliation) continue; owner.reconciliation.errors.push({ code: 'DNS_CLEANUP_FAILED', message: `Failed to delete ${record.type} ${record.name}: ${error.message}`, recordKey: record.managedRecordKey, retryable: true, }); owner.reconciliation.retryAt = new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString(); if (owner.reconciliation.lifecycleStatus !== 'deleting') { owner.reconciliation.lifecycleStatus = 'pending'; } await owner.save(); } } else { logger.log( 'error', `MailDnsSync failed to delete orphaned topology record ${record.type} ${record.name}: ${error.message}`, ); } } } private async pruneRetiredDkimMaterial(docs: EmailDomainDoc[]): Promise { const now = Date.now(); const records = await DnsRecordDoc.findAll(); for (const plannedDoc of docs) { const doc = await EmailDomainDoc.findById(plannedDoc.id); if (!doc) continue; const retiring = doc.retiringDkim || []; const retained = retiring.filter((material) => { const retireAt = Date.parse(material.retireAfter || ''); if (!Number.isFinite(retireAt) || retireAt > now) return true; const key = `mail-domain:${doc.id}:dkim:${material.selector}`; return records.some((record) => ( record.managedBy === MAIL_DNS_MANAGED_BY && record.managedOwnerId === `domain:${doc.id}` && record.managedRecordKey === key )); }); if (retained.length === retiring.length) continue; doc.retiringDkim = retained; doc.updatedAt = new Date().toISOString(); await doc.save(); } } private async finalizeDeletingDomains( deletingDocs: EmailDomainDoc[], zones: DomainDoc[], ): Promise { for (const deletingDoc of deletingDocs) { const doc = await EmailDomainDoc.findById(deletingDoc.id); if (!doc) continue; if (doc.reconciliation?.lifecycleStatus !== 'deleting') continue; if (doc.reconciliation.providerZoneIds === undefined) continue; if (doc.reconciliation.errors.some((error) => ( error.code === 'DNS_CLEANUP_FAILED' || error.code === 'DNS_PROVIDER_PROVENANCE_UNKNOWN' ))) continue; let providerProofComplete = true; for (const zoneId of doc.reconciliation.providerZoneIds) { const zone = zones.find((candidate) => candidate.id === zoneId); if (!zone || zone.source !== 'provider') { providerProofComplete = false; break; } const confirmation = await this.dcRouterRef.dnsManager!.syncDomain(zoneId); if (!confirmation.success) { providerProofComplete = false; doc.reconciliation.errors.push({ code: 'DNS_CLEANUP_FAILED', message: `Provider zone ${zone.name} could not confirm deletion: ${confirmation.message || 'list failed'}`, retryable: true, }); doc.reconciliation.retryAt = new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString(); await doc.save(); break; } const stillPublishedIds = new Set(confirmation.listedProviderRecordIds || []); const lingeringProviderId = (doc.reconciliation.deletingProviderRecordIds || []) .find((providerRecordId) => stillPublishedIds.has(providerRecordId)); if (lingeringProviderId) { providerProofComplete = false; doc.reconciliation.errors.push({ code: 'DNS_CLEANUP_FAILED', message: `Provider zone ${zone.name} still publishes deleting record ${lingeringProviderId}`, retryable: true, }); doc.reconciliation.retryAt = new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString(); await doc.save(); break; } } if (!providerProofComplete) continue; const records = await DnsRecordDoc.findAll(); const owned = records.some((record) => ( record.managedBy === MAIL_DNS_MANAGED_BY && record.managedOwnerId === `domain:${doc.id}` )); if (owned || doc.reconciliation.errors.some((error) => error.code === 'DNS_CLEANUP_FAILED')) continue; await doc.delete(); logger.log('info', `Email domain deletion finalized: ${doc.domain}`); } } private findZoneForName(name: string, zones: DomainDoc[]): DomainDoc | undefined { const normalized = normalizeHostname(name); return [...zones] .filter((zone) => normalized === normalizeHostname(zone.name) || normalized.endsWith(`.${normalizeHostname(zone.name)}`)) .sort((left, right) => right.name.length - left.name.length)[0]; } }