import { scheduler } from "node:timers/promises"; // "reset after 1h2m3s" / "10m15s" / "39s" const QUOTA_RESET_PATTERN = /reset after (?:(\d+)h)?(?:(\d+)m)?(\d+(?:\.\d+)?)s/i; // "Please retry in 250ms" / "Please retry in 12s" const PLEASE_RETRY_PATTERN = /Please retry in ([0-9.]+)(ms|s)/i; // JSON field: "retryDelay": "34.074824224s" const RETRY_DELAY_FIELD_PATTERN = /"retryDelay":\s*"([0-9.]+)(ms|s)"/i; // "try again in 250ms" / "try again in 12s" / "try again in 12sec" const TRY_AGAIN_PATTERN = /try again in\s+(\d+(?:\.\d+)?)\s*(ms|s)(?:ec)?/i; /** * Server-suggested retry delay extraction. Merges the patterns historically used * by the OpenAI code provider and Google Gemini retry helpers. * * Header sources (checked in order): * - `Retry-After` (numeric seconds, or HTTP date) * - `x-ratelimit-reset` (Unix epoch seconds) * - `x-ratelimit-reset-after` (seconds) * * Body patterns: * - `Your quota will reset after 18h31m10s` / `10m15s` / `39s` * - `Please retry in 250ms` / `Please retry in 12s` * - `"retryDelay": "34.074824224s"` (JSON error detail field) * - `try again in 250ms` / `try again in 12s` / `try again in 12sec` * * Returns `undefined` if no signal is found. */ export function extractRetryHint(source: Response | Headers | null | undefined, body?: string): number | undefined { const headers = source instanceof Headers ? source : (source?.headers ?? undefined); if (headers) { const retryAfter = headers.get("retry-after"); if (retryAfter) { const seconds = Number(retryAfter); if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000); const parsedDate = Date.parse(retryAfter); if (!Number.isNaN(parsedDate)) return Math.max(0, parsedDate - Date.now()); } const rateLimitReset = headers.get("x-ratelimit-reset"); if (rateLimitReset) { const resetSeconds = Number.parseInt(rateLimitReset, 10); if (!Number.isNaN(resetSeconds)) { const delta = resetSeconds * 1000 - Date.now(); if (delta > 0) return delta; } } const rateLimitResetAfter = headers.get("x-ratelimit-reset-after"); if (rateLimitResetAfter) { const seconds = Number(rateLimitResetAfter); if (Number.isFinite(seconds) && seconds > 0) return seconds * 1000; } } if (!body) return undefined; const quotaMatch = QUOTA_RESET_PATTERN.exec(body); if (quotaMatch) { const hours = quotaMatch[1] ? Number.parseInt(quotaMatch[1], 10) : 0; const minutes = quotaMatch[2] ? Number.parseInt(quotaMatch[2], 10) : 0; const seconds = Number.parseFloat(quotaMatch[3]!); if (!Number.isNaN(seconds)) { const totalMs = ((hours * 60 + minutes) * 60 + seconds) * 1000; if (totalMs > 0) return totalMs; } } for (const pattern of [PLEASE_RETRY_PATTERN, RETRY_DELAY_FIELD_PATTERN, TRY_AGAIN_PATTERN]) { const match = pattern.exec(body); if (match?.[1]) { const value = Number.parseFloat(match[1]); if (Number.isFinite(value) && value > 0) { return match[2]!.toLowerCase() === "ms" ? value : value * 1000; } } } return undefined; } export interface FetchWithRetryOptions extends RequestInit { /** Total fetch attempts (initial + retries). Default `5`. */ maxAttempts?: number; /** * Per-delay cap. Server-provided `Retry-After` hints exceeding this return * the current response immediately — caller deals with the `!response.ok`. * Scheduled values are also capped at the platform timer ceiling. Default * `60_000`. */ maxDelayMs?: number; /** * Fallback delay schedule when no server hint is present. Number, array * (indexed by attempt, clamped to last), or function. Default exponential * `500ms * 2 ** attempt` capped at `maxDelayMs` and the platform timer * ceiling. Values that remain negative or non-finite after capping retry * immediately. */ defaultDelayMs?: number | readonly number[] | ((attempt: number) => number); /** * Optional per-attempt overlay merged into the base `RequestInit` each try. * Headers from the overlay shallow-merge over the base. Useful for auth * token refresh or user-agent rotation. */ prepareInit?: (attempt: number) => RequestInit | Promise; /** * Optional `fetch` implementation override. Defaults to `globalThis.fetch`. * Useful for routing requests through a proxy, instrumented transport, or * mock during tests. */ fetch?: (input: string | URL | Request, init?: RequestInit) => Promise; } const DEFAULT_MAX_DELAY_MS = 60_000; const DEFAULT_MAX_ATTEMPTS = 5; // Node-compatible timers coerce larger delays to 1 ms. Bun follows the same // signed 32-bit boundary, so every scheduler input must stay at or below it. const MAX_TIMER_DELAY_MS = 2_147_483_647; /** * Fetch with bounded retries and sensible defaults. Retries on any * `isRetryableStatus` (5xx, 408, 429) and on transient network errors. Server * `Retry-After`/quota hints are honoured up to `maxDelayMs`; a hint that exceeds * the cap returns the current response so the caller can fail fast. Aborts on * `init.signal` propagate as `"Request was aborted"`. * * The caller is responsible for inspecting `!response.ok` once the call returns. */ export async function fetchWithRetry( url: string | URL | ((attempt: number) => string | URL), options: FetchWithRetryOptions = {}, ): Promise { const { maxAttempts = DEFAULT_MAX_ATTEMPTS, maxDelayMs = DEFAULT_MAX_DELAY_MS, defaultDelayMs, prepareInit, fetch: fetchImpl = fetch, ...baseInit } = options; const signal = baseInit.signal as AbortSignal | undefined; for (let attempt = 0; ; attempt++) { if (signal?.aborted) throw new Error("Request was aborted"); const requestUrl = typeof url === "function" ? url(attempt) : url; const init = prepareInit ? mergeInit(baseInit, await prepareInit(attempt)) : baseInit; let response: Response; try { response = await fetchImpl(requestUrl, init); } catch (error) { if (signal?.aborted) throw new Error("Request was aborted"); const wrapped = wrapNetworkError(error); if (attempt + 1 >= maxAttempts) throw wrapped; const delayMs = normalizeRetryDelay(resolveDefaultDelay(defaultDelayMs, attempt), maxDelayMs); await scheduler.wait(delayMs, { signal }); continue; } if (!isRetryableStatus(response.status)) return response; if (attempt + 1 >= maxAttempts) return response; const hint = extractRetryHint(response, await response.clone().text()); if (hint !== undefined && hint > maxDelayMs) return response; const delayMs = normalizeRetryDelay(hint ?? resolveDefaultDelay(defaultDelayMs, attempt), maxDelayMs); void response.body?.cancel().catch(() => undefined); await scheduler.wait(delayMs, { signal }); } } function mergeInit(base: RequestInit, overlay: RequestInit): RequestInit { const merged: RequestInit = { ...base, ...overlay }; if (base.headers || overlay.headers) { const baseHeaders = new Headers(base.headers ?? undefined); const overlayHeaders = new Headers(overlay.headers ?? undefined); overlayHeaders.forEach((value, key) => { baseHeaders.set(key, value); }); merged.headers = baseHeaders; } return merged; } function wrapNetworkError(error: unknown): Error { if (error instanceof Error) { if (error.name === "AbortError" || error.message === "Request was aborted") { return new Error("Request was aborted"); } if (error.message === "fetch failed" && error.cause instanceof Error) { return new Error(`Network error: ${error.cause.message}`); } return error; } return new Error(String(error)); } function resolveDefaultDelay(option: FetchWithRetryOptions["defaultDelayMs"], attempt: number): number { if (option === undefined) return 500 * 2 ** attempt; if (typeof option === "number") return option; if (typeof option === "function") return option(attempt); return option[Math.min(attempt, option.length - 1)] ?? 0; } function normalizeRetryDelay(delayMs: number, maxDelayMs: number): number { const cappedDelayMs = Math.min(delayMs, maxDelayMs, MAX_TIMER_DELAY_MS); return Number.isFinite(cappedDelayMs) && cappedDelayMs >= 0 ? cappedDelayMs : 0; } /** * Inspect an arbitrary error value (or its `cause` chain, up to depth 2) for an * HTTP status code. Reads `status`, `statusCode`, and `response.status` fields, * coerces string values, and falls back to scanning the error message for * common patterns like `Error: 401`, `error (429)`, or `HTTP 503`. */ export function extractHttpStatusFromError(error: unknown): number | undefined { return extractHttpStatusFromErrorInternal(error, 0); } type HttpErrorLike = { message?: string; name?: string; status?: number | string; statusCode?: number | string; response?: { status?: number | string }; cause?: unknown; }; function extractHttpStatusFromErrorInternal(error: unknown, depth: number): number | undefined { if (!error || typeof error !== "object" || depth > 2) return undefined; const info = error as HttpErrorLike; const rawStatus = info.status ?? info.statusCode ?? info.response?.status; let status: number | undefined; if (typeof rawStatus === "number" && Number.isFinite(rawStatus)) { status = rawStatus; } else if (typeof rawStatus === "string") { const parsed = Number(rawStatus); if (Number.isFinite(parsed)) status = parsed; } if (status !== undefined && status >= 100 && status <= 599) return status; if (info.message) { const extracted = extractStatusFromMessage(info.message); if (extracted !== undefined) return extracted; } if (info.cause) return extractHttpStatusFromErrorInternal(info.cause, depth + 1); return undefined; } const STATUS_MESSAGE_PATTERNS = [ /\berror\s*[:=]\s*(\d{3})\b/i, /error\s*\((\d{3})\)/i, /status\s*[:=]?\s*(\d{3})/i, /\bhttp\s*(\d{3})\b/i, /\b(\d{3})\s*(?:status|error)\b/i, ] as const; function extractStatusFromMessage(message: string): number | undefined { for (const pattern of STATUS_MESSAGE_PATTERNS) { const match = pattern.exec(message); if (!match) continue; const value = Number(match[1]); if (Number.isFinite(value) && value >= 100 && value <= 599) return value; } return undefined; } /** * `true` if the given HTTP status code is one we treat as transient: 408 * (Request Timeout), 429 (Too Many Requests), or any 5xx (server error). */ export function isRetryableStatus(status: number): boolean { return status >= 500 || status === 408 || status === 429; } /** * `true` if the message describes an unexpected socket closure — Bun and some * proxies surface these for any HTTP/2 stream reset. */ export function isUnexpectedSocketCloseMessage(message: string): boolean { return /\b(?:the\s+)?socket connection (?:was )?closed unexpectedly\b/i.test(message); } const TRANSIENT_MESSAGE_PATTERN = /overloaded|rate.?limit|too many requests|service.?unavailable|server error|internal error|connection.?error|unable to connect|fetch failed|network error|stream stall|other side closed/i; const VALIDATION_MESSAGE_PATTERN = /invalid|validation|bad request|unsupported|schema|missing required|not found|unauthorized|forbidden/i; /** * Identify errors that should be retried: aborts/timeouts in the error name or * message, retryable HTTP statuses (see `isRetryableStatus`), unexpected socket * closes, and the standard transient phrases. 4xx statuses other than 408/429 * and validation-shaped messages short-circuit to `false`. */ export function isRetryableError(error: unknown): boolean { const info = error as { message?: string; name?: string } | null; const message = info?.message ?? ""; const name = info?.name ?? ""; if (name === "AbortError" || /timeout|timed out|aborted/i.test(message)) return true; const status = extractHttpStatusFromError(error); if (status !== undefined) { if (isRetryableStatus(status)) return true; if (status >= 400 && status < 500) return false; } if (VALIDATION_MESSAGE_PATTERN.test(message)) return false; return isUnexpectedSocketCloseMessage(message) || TRANSIENT_MESSAGE_PATTERN.test(message); }