import { scheduler } from "node:timers/promises"; // "reset after 1h2m3s" / "Resets in 28h48m25.991s" / "10m15s" / "39s" const DURATION_TEXT_PATTERN = /([0-9.]+\s*(?:ms|h|m|s)(?:\s*[0-9.]+\s*(?:ms|h|m|s))*)/i; const QUOTA_RESET_PATTERN = new RegExp(`(?:reset after|resets?\\s+in)\\s+${DURATION_TEXT_PATTERN.source}`, "i"); // JSON fields: "retryDelay": "34.074824224s", "quotaResetDelay": "28h48m25.991228095s" const DURATION_FIELD_PATTERN = /"(?:retryDelay|quotaResetDelay)":\s*"([^"]+)"/i; // "Please retry in 250ms" / "Please retry in 12s" const PLEASE_RETRY_PATTERN = /Please retry in ([0-9.]+)(ms|s)/i; // "try again in 250ms" / "try again in 12s" / "try again in 12sec" / // "try again in 5 min" / "try again in ~158 min." / "try again in 2h" / // "try again in 90 minutes" / "try again in 1 hour" const TRY_AGAIN_PATTERN = /try again in\s+~?\s*([0-9.]+)\s*(ms|sec|s|minutes?|mins?|m|hours?|hrs?|h)\b/i; /** * Server-suggested retry delay extraction. Merges the patterns historically used * by the OpenAI Codex and Google Gemini retry helpers. * * Header sources (checked in order): * - `Retry-After` (numeric seconds, or HTTP date) * - `x-ratelimit-reset` (Unix epoch seconds) * - `x-ratelimit-reset-after` (seconds) * * Body patterns: * - `Your quota will reset after 18h31m10s` / `Resets in 28h48m25s` / `39s` * - `Please retry in 250ms` / `Please retry in 12s` * - `"retryDelay": "34.074824224s"` / `"quotaResetDelay": "28h48m25.991228095s"` * - `try again in 250ms` / `try again in 12s` / `try again in 5 min` / `try again in ~158 min` * * Returns `undefined` if no signal is found. */ export function extractRetryHint(source: Response | Headers | null | undefined, body?: string): number | undefined { const headers = source instanceof Headers ? source : (source?.headers ?? undefined); if (headers) { const retryAfter = headers.get("retry-after"); if (retryAfter) { const seconds = Number(retryAfter); if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000); const parsedDate = Date.parse(retryAfter); if (!Number.isNaN(parsedDate)) return Math.max(0, parsedDate - Date.now()); } const rateLimitReset = headers.get("x-ratelimit-reset"); if (rateLimitReset) { const resetSeconds = Number.parseInt(rateLimitReset, 10); if (!Number.isNaN(resetSeconds)) { const delta = resetSeconds * 1000 - Date.now(); if (delta > 0) return delta; } } const rateLimitResetAfter = headers.get("x-ratelimit-reset-after"); if (rateLimitResetAfter) { const seconds = Number(rateLimitResetAfter); if (Number.isFinite(seconds) && seconds > 0) return seconds * 1000; } } if (!body) return undefined; const quotaMatch = QUOTA_RESET_PATTERN.exec(body); if (quotaMatch?.[1]) { const totalMs = parseDurationTextMs(quotaMatch[1]); if (totalMs !== undefined) return totalMs; } const durationFieldMatch = DURATION_FIELD_PATTERN.exec(body); if (durationFieldMatch?.[1]) { const totalMs = parseDurationTextMs(durationFieldMatch[1]); if (totalMs !== undefined) return totalMs; } for (const pattern of [PLEASE_RETRY_PATTERN, TRY_AGAIN_PATTERN]) { const match = pattern.exec(body); if (match?.[1]) { const value = Number.parseFloat(match[1]); if (Number.isFinite(value) && value > 0) { const unitMs = unitToMs(match[2]!); if (unitMs !== undefined) return value * unitMs; } } } return undefined; } function parseDurationTextMs(value: string): number | undefined { let totalMs = 0; let matched = false; for (const match of value.matchAll(/([0-9.]+)\s*(ms|h|m|s)/gi)) { const amount = Number.parseFloat(match[1] ?? ""); const unit = match[2]; if (!Number.isFinite(amount) || !unit) continue; const unitMs = unitToMs(unit); if (unitMs === undefined) continue; totalMs += amount * unitMs; matched = true; } return matched && totalMs > 0 ? totalMs : undefined; } function unitToMs(unit: string): number | undefined { switch (unit.toLowerCase()) { case "ms": return 1; case "s": case "sec": return 1000; case "m": case "min": case "mins": case "minute": case "minutes": return 60_000; case "h": case "hr": case "hrs": case "hour": case "hours": return 60 * 60_000; default: return undefined; } } export interface FetchWithRetryOptions extends RequestInit { /** Total fetch attempts (initial + retries). Default `5`. */ maxAttempts?: number; /** * Per-delay cap. Server-provided `Retry-After` hints exceeding this return * the current response immediately — caller deals with the `!response.ok`. * Default `60_000`. */ maxDelayMs?: number; /** * Fallback delay schedule when no server hint is present. Number, array * (indexed by attempt, clamped to last), or function. Default exponential * `500ms * 2 ** attempt` capped at `maxDelayMs`. */ defaultDelayMs?: number | readonly number[] | ((attempt: number) => number); /** * Optional per-attempt overlay merged into the base `RequestInit` each try. * Headers from the overlay shallow-merge over the base. Useful for auth * token refresh or user-agent rotation. */ prepareInit?: (attempt: number) => RequestInit | Promise; /** * Optional `fetch` implementation override. Defaults to `globalThis.fetch`. * Useful for routing requests through a proxy, instrumented transport, or * mock during tests. */ fetch?: (input: string | URL | Request, init?: RequestInit) => Promise; /** * Optional retry gate for HTTP responses whose status is retryable. Receives a * cloned body string so callers can fail fast on deterministic provider * failures that happen to use a 5xx status. */ shouldRetryResponse?: (response: Response, bodyText: string, attempt: number) => boolean | Promise; /** * Bun extension forwarded verbatim to the underlying `fetch` call. `false` * disables Bun's native ~300s pre-response timeout (callers that own a * configurable first-event/idle watchdog or an external `AbortSignal` * supply this so the runtime ceiling cannot pre-empt them); a positive * number sets a custom ceiling in ms. Bare browser/Node fetch ignores it. */ timeout?: number | false; } const DEFAULT_MAX_DELAY_MS = 60_000; const DEFAULT_MAX_ATTEMPTS = 5; /** * Fetch with bounded retries and sensible defaults. Retries on any * `isRetryableStatus` (5xx, 408, 429) and on transient network errors. Server * `Retry-After`/quota hints are honoured up to `maxDelayMs`; a hint that exceeds * the cap returns the current response so the caller can fail fast. Aborts on * `init.signal` propagate as `"Request was aborted"`. * * The caller is responsible for inspecting `!response.ok` once the call returns. */ export async function fetchWithRetry( url: string | URL | ((attempt: number) => string | URL), options: FetchWithRetryOptions = {}, ): Promise { const { maxAttempts = DEFAULT_MAX_ATTEMPTS, maxDelayMs = DEFAULT_MAX_DELAY_MS, defaultDelayMs, prepareInit, shouldRetryResponse, fetch: fetchImpl = fetch, timeout = false, ...baseInit } = options; const signal = baseInit.signal as AbortSignal | undefined; for (let attempt = 0; ; attempt++) { if (signal?.aborted) throw new Error("Request was aborted"); const requestUrl = typeof url === "function" ? url(attempt) : url; // `timeout` is destructured out of `baseInit`, so forward it to the underlying // fetch on the no-`prepareInit` path too. Without this, callers that pass // `timeout: false` (every streaming provider, to disable Bun's native ~300s // fetch ceiling in favor of their own first-event/idle watchdog) had it // silently dropped, so long-running streams were killed at ~300s (issue #602). // Only forward when the caller actually set `timeout`, so callers that never // set it keep Bun's default ceiling. const init = prepareInit ? mergeInit(baseInit, await prepareInit(attempt), timeout) : "timeout" in options ? ({ ...baseInit, timeout } as unknown as RequestInit) : baseInit; let response: Response; try { response = await fetchImpl(requestUrl, init); } catch (error) { if (signal?.aborted) throw new Error("Request was aborted"); const wrapped = wrapNetworkError(error); if (attempt + 1 >= maxAttempts) throw wrapped; await scheduler.wait(resolveDefaultDelay(defaultDelayMs, attempt, maxDelayMs), { signal }); continue; } if (!isRetryableStatus(response.status)) return response; if (attempt + 1 >= maxAttempts) return response; const retryBody = await response.clone().text(); if (shouldRetryResponse && !(await shouldRetryResponse(response, retryBody, attempt))) return response; const hint = extractRetryHint(response, retryBody); if (hint !== undefined && hint > maxDelayMs) return response; const delayMs = Math.min(hint ?? resolveDefaultDelay(defaultDelayMs, attempt, maxDelayMs), maxDelayMs); await scheduler.wait(delayMs, { signal }); } } function mergeInit(base: RequestInit, overlay: RequestInit, timeout: number | false): RequestInit { const merged = { ...base, ...overlay, timeout } as unknown as RequestInit; if (base.headers || overlay.headers) { const baseHeaders = new Headers(base.headers ?? undefined); const overlayHeaders = new Headers(overlay.headers ?? undefined); overlayHeaders.forEach((value, key) => { baseHeaders.set(key, value); }); merged.headers = baseHeaders; } return merged; } function wrapNetworkError(error: unknown): Error { if (error instanceof Error) { if (error.name === "AbortError" || error.message === "Request was aborted") { return new Error("Request was aborted"); } if (error.message === "fetch failed" && error.cause instanceof Error) { return new Error(`Network error: ${error.cause.message}`); } return error; } return new Error(String(error)); } function resolveDefaultDelay( option: FetchWithRetryOptions["defaultDelayMs"], attempt: number, maxDelayMs: number, ): number { if (option === undefined) return Math.min(500 * 2 ** attempt, maxDelayMs); if (typeof option === "number") return Math.min(option, maxDelayMs); if (typeof option === "function") return Math.min(option(attempt), maxDelayMs); return Math.min(option[Math.min(attempt, option.length - 1)] ?? 0, maxDelayMs); } /** * Inspect an arbitrary error value (or its `cause` chain, up to depth 2) for an * HTTP status code. Reads `status`, `statusCode`, and `response.status` fields, * coerces string values, and falls back to scanning the error message for * common patterns like `Error: 401`, `error (429)`, or `HTTP 503`. */ export function extractHttpStatusFromError(error: unknown): number | undefined { return extractHttpStatusFromErrorInternal(error, 0); } type HttpErrorLike = { message?: string; name?: string; status?: number | string; statusCode?: number | string; response?: { status?: number | string }; cause?: unknown; }; function extractHttpStatusFromErrorInternal(error: unknown, depth: number): number | undefined { if (!error || typeof error !== "object" || depth > 2) return undefined; const info = error as HttpErrorLike; const rawStatus = info.status ?? info.statusCode ?? info.response?.status; let status: number | undefined; if (typeof rawStatus === "number" && Number.isFinite(rawStatus)) { status = rawStatus; } else if (typeof rawStatus === "string") { const parsed = Number(rawStatus); if (Number.isFinite(parsed)) status = parsed; } if (status !== undefined && status >= 100 && status <= 599) return status; if (info.message) { const extracted = extractStatusFromMessage(info.message); if (extracted !== undefined) return extracted; } if (info.cause) return extractHttpStatusFromErrorInternal(info.cause, depth + 1); return undefined; } const STATUS_MESSAGE_PATTERNS = [ /\berror\s*[:=]\s*(\d{3})\b/i, /error\s*\((\d{3})\)/i, /status\s*[:=]?\s*(\d{3})/i, /\bhttp\s*(\d{3})\b/i, /\b(\d{3})\s*(?:status|error)\b/i, ] as const; function extractStatusFromMessage(message: string): number | undefined { for (const pattern of STATUS_MESSAGE_PATTERNS) { const match = pattern.exec(message); if (!match) continue; const value = Number(match[1]); if (Number.isFinite(value) && value >= 100 && value <= 599) return value; } return undefined; } /** * `true` if the given HTTP status code is one we treat as transient: 408 * (Request Timeout), 429 (Too Many Requests), or any 5xx (server error). */ export function isRetryableStatus(status: number): boolean { return status >= 500 || status === 408 || status === 429; } /** * `true` if the message describes an unexpected socket closure — Bun and some * proxies surface these for any HTTP/2 stream reset. */ export function isUnexpectedSocketCloseMessage(message: string): boolean { return /\b(?:the\s+)?socket connection (?:was )?closed unexpectedly\b/i.test(message); } const TRANSIENT_MESSAGE_PATTERN = /overloaded|rate.?limit|too many requests|service.?unavailable|server error|internal error|connection.?error|unable to connect|fetch failed|network error|stream stall|other side closed|HTTP2(?:StreamReset|RefusedStream|EnhanceYourCalm)/i; const VALIDATION_MESSAGE_PATTERN = /invalid|validation|bad request|unsupported|schema|missing required|not found|unauthorized|forbidden/i; /** * Identify errors that should be retried: aborts/timeouts in the error name or * message, retryable HTTP statuses (see `isRetryableStatus`), unexpected socket * closes, and the standard transient phrases. 4xx statuses other than 408/429 * and validation-shaped messages short-circuit to `false`. */ export function isRetryableError(error: unknown): boolean { const info = error as { message?: string; name?: string } | null; const message = info?.message ?? ""; const name = info?.name ?? ""; if (name === "AbortError" || /timeout|timed out|aborted/i.test(message)) return true; const status = extractHttpStatusFromError(error); if (status !== undefined) { if (isRetryableStatus(status)) return true; if (status >= 400 && status < 500) return false; } if (VALIDATION_MESSAGE_PATTERN.test(message)) return false; return isUnexpectedSocketCloseMessage(message) || TRANSIENT_MESSAGE_PATTERN.test(message); }