/** * Tiny wrapper around global `fetch` with: * - per-call timeout (AbortSignal.timeout) * - retry on transient failure (5xx / 429 / network error) * - structured error → MemosError(code=embedding_unavailable) * * Providers should never call `fetch` directly; go through `httpPostJson`. */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { getRetryCooldown, parseRetryAfterMs, planRetry, recordRetryCooldown, retryCooldownKey, type RetryPlan, waitForRetry, } from "../util/retry-after.js"; import type { EmbeddingProviderName, ProviderLogger } from "./types.js"; export interface HttpPostOpts { url: string; body: TBody; headers?: Record; timeoutMs?: number; maxRetries?: number; signal?: AbortSignal; /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ deadlineAt?: number; /** Model/deployment scope; prevents one model cooldown from blocking another. */ cooldownScope?: string; provider: EmbeddingProviderName; log: ProviderLogger; } export async function httpPostJson(opts: HttpPostOpts): Promise { const timeoutMs = opts.timeoutMs ?? 30_000; const maxRetries = opts.maxRetries ?? 2; let attempt = 0; let lastErr: unknown = null; const cooldownKey = retryCooldownKey("embedding", opts.provider, opts.url, opts.cooldownScope); while (attempt <= maxRetries) { attempt++; const start = Date.now(); try { const cooldown = getRetryCooldown(cooldownKey, start); if (cooldown) { const details = { provider: opts.provider, url: opts.url, status: cooldown.status, attempt, maxRetries, retryAfterMs: cooldown.retryAfterMs, retryAt: cooldown.retryAt, retryDecision: "defer", retryReason: "cooldown_active", remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), }; opts.log.warn("http.retry_cooldown", details); throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, details, ); } const signal = mergeSignals(opts.signal, AbortSignal.timeout(timeoutMs)); const resp = await fetch(opts.url, { method: "POST", headers: { "Content-Type": "application/json", Accept: "application/json", ...opts.headers, }, body: JSON.stringify(opts.body), signal, }); if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; const retryAfterMs = resp.status === 429 || resp.status === 503 ? parseRetryAfterMs(resp.headers.get("Retry-After")) : null; if (retryAfterMs !== null) { recordRetryCooldown(cooldownKey, { retryAfterMs, retryAt: Date.now() + retryAfterMs, status: resp.status, }); } opts.log.warn("http.non_ok", { url: opts.url, status: resp.status, attempt, transient, retryAfterMs, durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { const plan = planRetry({ attempt, baseMs: 200, jitterMaxMs: 100, retryAfterMs, deadlineAt: opts.deadlineAt, }); const retryDetails = retryPlanDetails(plan, opts, maxRetries, resp.status, attempt); if (plan.action === "defer") { opts.log.warn("http.retry_deferred", retryDetails); throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, retryDetails, ); } opts.log.warn("http.retry_scheduled", retryDetails); await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `HTTP ${resp.status} from ${opts.provider}`, { provider: opts.provider, url: opts.url, status: resp.status, body: text, ...(retryAfterMs === null ? {} : { retryAfterMs, retryAt: Date.now() + retryAfterMs, retryDecision: "stop", retryReason: "retries_exhausted", }), }, ); } opts.log.debug("http.ok", { url: opts.url, status: resp.status, attempt, durationMs: Date.now() - start, }); return (await resp.json()) as TResp; } catch (err) { lastErr = err; if (err instanceof MemosError) throw err; const transient = isTransientError(err); opts.log.warn("http.exception", { url: opts.url, attempt, transient, err: serializeErr(err), durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { const plan = planRetry({ attempt, baseMs: 200, jitterMaxMs: 100, deadlineAt: opts.deadlineAt, }); const retryDetails = retryPlanDetails(plan, opts, maxRetries, null, attempt); if (plan.action === "defer") { opts.log.warn("http.retry_deferred", retryDetails); throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `${opts.provider} retry cannot fit the request deadline`, retryDetails, ); } opts.log.warn("http.retry_scheduled", retryDetails); await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `Network error calling ${opts.provider}: ${(err as Error).message ?? String(err)}`, { provider: opts.provider, url: opts.url }, ); } } throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `Exhausted retries to ${opts.provider}`, { provider: opts.provider, url: opts.url, cause: lastErr instanceof Error ? lastErr.message : String(lastErr), }, ); } async function safeText(resp: Response): Promise { try { return await resp.text(); } catch { return undefined; } } function isTransientError(err: unknown): boolean { if (!(err instanceof Error)) return false; // Node fetch maps network errors to specific causes; abort with timeout is // also retriable once. We're conservative here. const msg = err.message ?? ""; if (/timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|socket hang up/i.test(msg)) return true; if ((err as { code?: string }).code === "ABORT_ERR") return true; return false; } function retryPlanDetails( plan: RetryPlan, opts: HttpPostOpts, maxRetries: number, status: number | null, attempt: number, ): Record { return { provider: opts.provider, url: opts.url, status, attempt, maxRetries, backoffMs: plan.backoffMs, plannedDelayMs: plan.delayMs, retryAfterMs: plan.retryAfterMs, retryAt: plan.retryAt, retrySource: plan.source, retryDecision: plan.action, ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), }; } function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { if (!a) return b; const ctrl = new AbortController(); const forward = () => ctrl.abort(); if (a.aborted || b.aborted) ctrl.abort(); a.addEventListener("abort", forward, { once: true }); b.addEventListener("abort", forward, { once: true }); return ctrl.signal; } function serializeErr(e: unknown): Record { if (e instanceof Error) { return { name: e.name, message: e.message }; } return { value: String(e) }; }