import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; import type { DcRouter } from '../classes.dcrouter.js'; import { AcmePermanentFailureError, classifyAcmeFailure, type TAcmeFailureReason, } from './acme-failure-classification.js'; /** Terminal state of the SmartAcme provider startup budget. */ export interface IAcmeStartFailure { reason: TAcmeFailureReason; /** True when no further retry can resolve the cause. */ permanent: boolean; /** Attempts consumed when the budget ended. */ attempts: number; message: string; at: number; /** What an operator has to do to re-arm startup. */ rearmedBy: string; } /** Attempts allowed for transient SmartAcme startup failures before giving up. */ export const smartAcmeStartMaxAttempts = 20; /** * Background start/retry/stop lifecycle for the DcRouter-owned SmartAcme * instance. SmartAcme startup can hit ACME rate limits, so startup runs in * the background with generation-guarded exponential retry, and certificate * provisioning is re-triggered once DNS-01 becomes ready. * * Retry semantics, which are NOT shared with the per-domain * `CertProvisionScheduler` budget: * - increments once per failed `smartAcme.start()`; * - 5 s → 1 h exponential backoff with ±20% jitter; * - capped at `smartAcmeStartMaxAttempts` transient attempts; * - reset by `startInBackground()`, `stop()`, and a successful start; * - re-armed only by `startInBackground()`, i.e. a SmartProxy rebuild, an * explicit `rearm()` after a configuration change, or a process restart. * Nothing re-arms it on a timer, so the terminal state is recorded in * `startFailure` and logged at `error` instead of scrolling past as a warning. * * A permanent cause never enters the backoff at all: retrying a misconfigured * ACME account cannot fix it, and spending the budget on it hides the reason. */ export class SmartAcmeLifecycle { /** True once the SmartAcme DNS-01 provider finished starting. */ public ready = false; /** Tracks whether the taskbuffer SmartAcme service is started, so SmartProxy rebuilds can re-kick startup. */ public serviceStarted = false; /** * Set when the startup budget ended — either immediately for a permanent cause * or after the transient attempt cap. Cleared on every (re-)arm and on success. */ public startFailure?: IAcmeStartFailure; private startGeneration = 0; private startPromise?: Promise; private retryTimer?: ReturnType; private retryAttempt = 0; constructor(private dcRouterRef: DcRouter) {} public startInBackground(): void { if (!this.dcRouterRef.smartAcme) { this.ready = false; return; } const generation = ++this.startGeneration; this.ready = false; this.retryAttempt = 0; this.startFailure = undefined; this.clearRetryTimer(); this.scheduleStart(generation, 0); } /** * Re-arm startup after a configuration change that can plausibly fix a * previously terminal cause (e.g. the ACME account settings were corrected). * Without this, an exhausted or permanently-failed budget could only be reset * by a SmartProxy rebuild or a full restart — and a dcrouter restart is a * measured 30–60 s of total public outage. */ public rearm(reasonArg: string): boolean { if (!this.dcRouterRef.smartAcme || !this.serviceStarted) { return false; } if (this.ready && !this.startFailure) { return false; } logger.log('info', `Re-arming SmartAcme DNS-01 provider startup: ${reasonArg}`); this.startInBackground(); return true; } public async stop(): Promise { this.startGeneration++; this.ready = false; this.retryAttempt = 0; this.startFailure = undefined; this.clearRetryTimer(); const smartAcme = this.dcRouterRef.smartAcme; if (!smartAcme) { return; } try { await smartAcme.stop(); } catch (err) { logger.log('error', 'Error stopping SmartAcme', { error: String(err) }); } finally { if (this.dcRouterRef.smartAcme === smartAcme) { this.dcRouterRef.smartAcme = undefined; } } } private scheduleStart(generation: number, delayMs: number): void { this.clearRetryTimer(); const retryTimer = setTimeout(() => { this.retryTimer = undefined; this.runStartAttempt(generation).catch((err) => { logger.log('error', `Unexpected SmartAcme startup error: ${(err as Error).message}`); }); }, delayMs); this.retryTimer = retryTimer; const unrefableTimer = retryTimer as any; if (typeof unrefableTimer?.unref === 'function') { unrefableTimer.unref(); } } private async runStartAttempt(generation: number): Promise { const smartAcme = this.dcRouterRef.smartAcme; if (!smartAcme || generation !== this.startGeneration) { return; } const startPromise = smartAcme.start(); this.startPromise = startPromise; try { await startPromise; if (generation !== this.startGeneration || this.dcRouterRef.smartAcme !== smartAcme) { await smartAcme.stop().catch((err) => { logger.log('warn', `Failed to stop stale SmartAcme instance: ${(err as Error).message}`); }); return; } this.ready = true; this.retryAttempt = 0; this.startFailure = undefined; logger.log('info', 'SmartAcme DNS-01 provider is now ready'); this.retriggerCertificateProvisioning(); } catch (err) { if (generation !== this.startGeneration || this.dcRouterRef.smartAcme !== smartAcme) { return; } this.ready = false; await smartAcme.stop().catch((stopErr) => { logger.log('warn', `Failed to clean up SmartAcme after startup failure: ${(stopErr as Error).message}`); }); // Classify before spending any of the budget: a permanent cause is not made // truer by 20 more attempts, and every attempt spent on it is an hour in // which nothing tells the operator what is actually wrong. const classification = classifyAcmeFailure(err); if (classification.permanent) { this.startFailure = { reason: classification.reason, permanent: true, attempts: this.retryAttempt, message: classification.message, at: Date.now(), rearmedBy: 'fixing the ACME configuration (which calls rearm()), a SmartProxy rebuild, or a process restart', }; // Constructing this logs at `error` with the code and category attached. new AcmePermanentFailureError( classification, 'SmartAcme DNS-01 provider startup', 'smartacme-lifecycle', { data: { attempts: this.retryAttempt, rearmedBy: this.startFailure.rearmedBy } }, ); return; } this.retryAttempt++; if (this.retryAttempt > smartAcmeStartMaxAttempts) { this.startFailure = { reason: classification.reason, permanent: false, attempts: this.retryAttempt - 1, message: classification.message, at: Date.now(), rearmedBy: 'a SmartProxy rebuild, an explicit rearm() after a configuration change, or a process restart', }; logger.log( 'error', `SmartAcme DNS-01 provider gave up after ${smartAcmeStartMaxAttempts} startup attempts ` + `(${classification.reason}): ${classification.message}. ` + 'Nothing re-arms this on a timer — DNS-01 stays unavailable and every affected route falls back to a ' + `no-op http-01 path until ${this.startFailure.rearmedBy}.`, { acmeFailureReason: classification.reason, attempts: this.startFailure.attempts }, ); return; } const baseDelayMs = 5000; const maxDelayMs = 3_600_000; const delayMs = Math.min(baseDelayMs * Math.pow(2, this.retryAttempt - 1), maxDelayMs); const jitter = 0.8 + Math.random() * 0.4; const actualDelayMs = Math.floor(delayMs * jitter); logger.log('warn', `SmartAcme DNS-01 provider startup failed: ${(err as Error).message}; retrying in ${actualDelayMs}ms (attempt ${this.retryAttempt}/${smartAcmeStartMaxAttempts})`); this.scheduleStart(generation, actualDelayMs); } finally { if (this.startPromise === startPromise) { this.startPromise = undefined; } } } private retriggerCertificateProvisioning(): void { // During startup, certProvisionFunction returns 'http01' while SmartAcme is not ready, // but Rust ACME is disabled when certProvisionFunction is set. Re-applying routes // retries provisioning now that DNS-01 is available. if (this.dcRouterRef.routeConfigManager) { logger.log('info', 'Re-triggering certificate provisioning via RouteConfigManager'); this.dcRouterRef.routeConfigManager.applyRoutes().catch((err: any) => { logger.log('warn', `Failed to re-trigger cert provisioning: ${err?.message || err}`); }); return; } if (this.dcRouterRef.smartProxy) { if (this.dcRouterRef.certProvisionScheduler) { this.dcRouterRef.certProvisionScheduler.clear(); } const currentRoutes = this.dcRouterRef.smartProxy.routeManager.getRoutes(); logger.log('info', `Re-triggering certificate provisioning for ${currentRoutes.length} routes`); this.dcRouterRef.smartProxy.updateRoutes(currentRoutes).catch((err: any) => { logger.log('warn', `Failed to re-trigger cert provisioning: ${err?.message || err}`); }); } } private clearRetryTimer(): void { if (this.retryTimer) { clearTimeout(this.retryTimer); this.retryTimer = undefined; } } }