import { ActorError, HttpRequestError } from "./errors"; import { type RetryBudget, retryAfterMs, retryDelayMs } from "./retry-budget"; export interface LifecycleBoundaryInfo { kind: "request_retry" | "reconnect_only"; source: "actor_error" | "transport_error"; group?: string; code?: string; message: string; legacy: boolean; } function* walkErrorChain( error: unknown, maxDepth = 8, ): Generator { let current = error; let depth = 0; while (current !== undefined && current !== null && depth < maxDepth) { yield current; if ( typeof current === "object" && "cause" in current && (current as { cause?: unknown }).cause !== current ) { current = (current as { cause?: unknown }).cause; depth += 1; continue; } break; } } function buildLifecycleBoundaryInfo( kind: LifecycleBoundaryInfo["kind"], source: LifecycleBoundaryInfo["source"], message: string, opts?: { group?: string; code?: string; legacy?: boolean; }, ): LifecycleBoundaryInfo { return { kind, source, group: opts?.group, code: opts?.code, message, legacy: opts?.legacy ?? false, }; } function classifyActorError( error: ActorError, ): LifecycleBoundaryInfo | undefined { if ( error.group === "actor" && error.code === "stopping" && error.message.includes("database accessed after actor stopped") ) { return undefined; } // `actor.migrating_from_v2` is a refusal while the actor moves in from a V2 cluster; it // carries a `retry_after_ms` hint. if ( error.group === "actor" && (error.code === "restarting" || error.code === "migrating_from_v2") ) { return buildLifecycleBoundaryInfo( "request_retry", "actor_error", error.message, { group: error.group, code: error.code, }, ); } if ( (error.group === "ups" && error.code === "consumer_closed") || (error.group === "depot" && error.code === "request_deadline_exceeded") ) { // The connection can recover, but an interrupted action may already have run. return buildLifecycleBoundaryInfo( "reconnect_only", "actor_error", error.message, { group: error.group, code: error.code }, ); } if ( error.group === "guard" && isRetryableGuardGatewayHttpError(error.code) ) { return buildLifecycleBoundaryInfo( "request_retry", "actor_error", error.message, { group: error.group, code: error.code, }, ); } // TODO(RVT-6193): Remove this legacy match after structured restart errors // are authoritative everywhere. if ( error.group === "actor" && error.code === "internal_error" && error.message === "Actor is stopping" ) { return buildLifecycleBoundaryInfo( "request_retry", "actor_error", error.message, { group: error.group, code: error.code, legacy: true, }, ); } // TODO(RVT-6193): Remove this legacy match after connection admission uses // actor.restarting consistently. if ( error.group === "actor" && error.code === "stopping" && error.message === "Actor stopping: Cannot accept new connections while actor is stopping" ) { return buildLifecycleBoundaryInfo( "request_retry", "actor_error", error.message, { group: error.group, code: error.code, legacy: true, }, ); } if (error.group === "actor" && error.code === "stopped") { return buildLifecycleBoundaryInfo( "reconnect_only", "actor_error", error.message, { group: error.group, code: error.code, legacy: true, }, ); } if (error.group === "ws" && error.code === "going_away") { return buildLifecycleBoundaryInfo( "reconnect_only", "actor_error", error.message, { group: error.group, code: error.code, legacy: true, }, ); } return undefined; } function isRetryableGuardGatewayHttpError(code: string): boolean { return ( code === "service_unavailable" || code === "actor_wake_retries_exceeded" || code === "actor_stopped_while_waiting" || code === "tunnel_request_aborted" || code === "tunnel_message_timeout" || code === "tunnel_response_closed" || code === "gateway_response_start_timeout" ); } function classifyTransportError( error: Error, ): LifecycleBoundaryInfo | undefined { if (error.message.includes("database accessed after actor stopped")) { return undefined; } // TODO(RVT-6193): Remove this exact string match after the runner surfaces // structured restart errors end to end. if (/^Actor [A-Za-z0-9-]+ stopped$/.test(error.message)) { return buildLifecycleBoundaryInfo( "request_retry", "transport_error", error.message, { legacy: true }, ); } // TODO(RVT-6193): Remove these exact string matches after transport // shutdown paths are normalized to structured lifecycle errors. if ( error.message === "WebSocket connection closed during shutdown" || error.message === "envoy shut down" || error.message === "envoy shutting down" ) { return buildLifecycleBoundaryInfo( "reconnect_only", "transport_error", error.message, { legacy: true }, ); } return undefined; } export function classifyLifecycleBoundaryError( error: unknown, ): LifecycleBoundaryInfo | undefined { for (const current of walkErrorChain(error)) { if (current instanceof ActorError) { const classified = classifyActorError(current); if (classified) { return classified; } continue; } if (current instanceof HttpRequestError || current instanceof Error) { const classified = classifyTransportError(current); if (classified) { return classified; } } } return undefined; } export function isRetryableLifecycleRequestError(error: unknown): boolean { return classifyLifecycleBoundaryError(error)?.kind === "request_retry"; } export function isRetryableLifecycleReconnectSignal(error: unknown): boolean { const classified = classifyLifecycleBoundaryError(error); return ( classified?.kind === "reconnect_only" || classified?.kind === "request_retry" ); } function throwIfAborted(signal?: AbortSignal) { if (signal?.aborted) { throw signal.reason ?? new Error("Operation aborted"); } } async function waitWithSignal(ms: number, signal?: AbortSignal) { throwIfAborted(signal); await new Promise((resolve, reject) => { const timeout = setTimeout(() => { cleanup(); resolve(); }, ms); const onAbort = () => { clearTimeout(timeout); cleanup(); reject(signal?.reason ?? new Error("Operation aborted")); }; const cleanup = () => { signal?.removeEventListener("abort", onAbort); }; signal?.addEventListener("abort", onAbort, { once: true }); }); } /** * Whether a fork was refused before it did anything, so sending it again cannot make a second * copy: `actor.fork_unavailable` while an engine upgrade rolls out, and * `guard.service_unavailable` with the reason `actor_moving` while a keyed source waits for its * namespace's backfill. Every other fork error, including the timeouts after which a copy may * exist, goes to the caller. */ export function isRetryableForkRefusal(error: unknown): boolean { if (!(error instanceof ActorError)) { return false; } if (error.group === "actor" && error.code === "fork_unavailable") { return true; } if (error.group === "guard" && error.code === "service_unavailable") { const metadata = error.metadata as | { reason?: unknown } | null | undefined; return metadata?.reason === "actor_moving"; } return false; } /** * Runs `run`, retrying retryable lifecycle and refusal errors with jittered exponential backoff * that honors the service's `retry_after_ms` hint. With a `budget`, the call credits it once and * every retry spends a token; a spent budget returns the error instead of retrying. * `isRetryable` replaces the lifecycle classification for calls that retry other errors. */ export async function retryOnLifecycleBoundary( run: () => Promise, opts?: { maxAttempts?: number; initialDelayMs?: number; maxDelayMs?: number; signal?: AbortSignal; budget?: RetryBudget; isRetryable?: (error: unknown) => boolean; }, ): Promise { const maxAttempts = opts?.maxAttempts ?? 5; const initialDelayMs = opts?.initialDelayMs ?? 25; const maxDelayMs = opts?.maxDelayMs ?? 200; const isRetryable = opts?.isRetryable ?? isRetryableLifecycleRequestError; opts?.budget?.deposit(); let lastError: unknown; for (let attempt = 0; attempt < maxAttempts; attempt += 1) { throwIfAborted(opts?.signal); try { return await run(); } catch (error) { if (!isRetryable(error)) { throw error; } lastError = error; if (attempt === maxAttempts - 1) { break; } if (opts?.budget && !opts.budget.tryWithdraw()) { break; } const delayMs = retryDelayMs(attempt, { initialDelayMs, maxDelayMs, hintMs: retryAfterMs(error), }); await waitWithSignal(delayMs, opts?.signal); } } throw lastError; }