/**
* Tiny wrapper around global `fetch` with:
* - per-call timeout (AbortSignal.timeout)
* - retry on transient failure (5xx / 429 / network error)
* - structured error → MemosError(code=embedding_unavailable)
*
* Providers should never call `fetch` directly; go through `httpPostJson`.
*/
import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js";
import {
getRetryCooldown,
parseRetryAfterMs,
planRetry,
recordRetryCooldown,
retryCooldownKey,
type RetryPlan,
waitForRetry,
} from "../util/retry-after.js";
import type { EmbeddingProviderName, ProviderLogger } from "./types.js";
export interface HttpPostOpts
{
url: string;
body: TBody;
headers?: Record;
timeoutMs?: number;
maxRetries?: number;
signal?: AbortSignal;
/** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */
deadlineAt?: number;
/** Model/deployment scope; prevents one model cooldown from blocking another. */
cooldownScope?: string;
provider: EmbeddingProviderName;
log: ProviderLogger;
}
export async function httpPostJson(opts: HttpPostOpts): Promise {
const timeoutMs = opts.timeoutMs ?? 30_000;
const maxRetries = opts.maxRetries ?? 2;
let attempt = 0;
let lastErr: unknown = null;
const cooldownKey = retryCooldownKey("embedding", opts.provider, opts.url, opts.cooldownScope);
while (attempt <= maxRetries) {
attempt++;
const start = Date.now();
try {
const cooldown = getRetryCooldown(cooldownKey, start);
if (cooldown) {
const details = {
provider: opts.provider,
url: opts.url,
status: cooldown.status,
attempt,
maxRetries,
retryAfterMs: cooldown.retryAfterMs,
retryAt: cooldown.retryAt,
retryDecision: "defer",
retryReason: "cooldown_active",
remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start),
};
opts.log.warn("http.retry_cooldown", details);
throw new MemosError(
ERROR_CODES.EMBEDDING_UNAVAILABLE,
`${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`,
details,
);
}
const signal = mergeSignals(opts.signal, AbortSignal.timeout(timeoutMs));
const resp = await fetch(opts.url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
...opts.headers,
},
body: JSON.stringify(opts.body),
signal,
});
if (!resp.ok) {
const text = await safeText(resp);
const transient = resp.status >= 500 || resp.status === 429;
const retryAfterMs = resp.status === 429 || resp.status === 503
? parseRetryAfterMs(resp.headers.get("Retry-After"))
: null;
if (retryAfterMs !== null) {
recordRetryCooldown(cooldownKey, {
retryAfterMs,
retryAt: Date.now() + retryAfterMs,
status: resp.status,
});
}
opts.log.warn("http.non_ok", {
url: opts.url,
status: resp.status,
attempt,
transient,
retryAfterMs,
durationMs: Date.now() - start,
});
if (transient && attempt <= maxRetries) {
const plan = planRetry({
attempt,
baseMs: 200,
jitterMaxMs: 100,
retryAfterMs,
deadlineAt: opts.deadlineAt,
});
const retryDetails = retryPlanDetails(plan, opts, maxRetries, resp.status, attempt);
if (plan.action === "defer") {
opts.log.warn("http.retry_deferred", retryDetails);
throw new MemosError(
ERROR_CODES.EMBEDDING_UNAVAILABLE,
`HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`,
retryDetails,
);
}
opts.log.warn("http.retry_scheduled", retryDetails);
await waitForRetry(plan.delayMs, opts.signal);
continue;
}
throw new MemosError(
ERROR_CODES.EMBEDDING_UNAVAILABLE,
`HTTP ${resp.status} from ${opts.provider}`,
{
provider: opts.provider,
url: opts.url,
status: resp.status,
body: text,
...(retryAfterMs === null
? {}
: {
retryAfterMs,
retryAt: Date.now() + retryAfterMs,
retryDecision: "stop",
retryReason: "retries_exhausted",
}),
},
);
}
opts.log.debug("http.ok", {
url: opts.url,
status: resp.status,
attempt,
durationMs: Date.now() - start,
});
return (await resp.json()) as TResp;
} catch (err) {
lastErr = err;
if (err instanceof MemosError) throw err;
const transient = isTransientError(err);
opts.log.warn("http.exception", {
url: opts.url,
attempt,
transient,
err: serializeErr(err),
durationMs: Date.now() - start,
});
if (transient && attempt <= maxRetries) {
const plan = planRetry({
attempt,
baseMs: 200,
jitterMaxMs: 100,
deadlineAt: opts.deadlineAt,
});
const retryDetails = retryPlanDetails(plan, opts, maxRetries, null, attempt);
if (plan.action === "defer") {
opts.log.warn("http.retry_deferred", retryDetails);
throw new MemosError(
ERROR_CODES.EMBEDDING_UNAVAILABLE,
`${opts.provider} retry cannot fit the request deadline`,
retryDetails,
);
}
opts.log.warn("http.retry_scheduled", retryDetails);
await waitForRetry(plan.delayMs, opts.signal);
continue;
}
throw new MemosError(
ERROR_CODES.EMBEDDING_UNAVAILABLE,
`Network error calling ${opts.provider}: ${(err as Error).message ?? String(err)}`,
{ provider: opts.provider, url: opts.url },
);
}
}
throw new MemosError(
ERROR_CODES.EMBEDDING_UNAVAILABLE,
`Exhausted retries to ${opts.provider}`,
{
provider: opts.provider,
url: opts.url,
cause: lastErr instanceof Error ? lastErr.message : String(lastErr),
},
);
}
async function safeText(resp: Response): Promise {
try {
return await resp.text();
} catch {
return undefined;
}
}
function isTransientError(err: unknown): boolean {
if (!(err instanceof Error)) return false;
// Node fetch maps network errors to specific causes; abort with timeout is
// also retriable once. We're conservative here.
const msg = err.message ?? "";
if (/timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|socket hang up/i.test(msg)) return true;
if ((err as { code?: string }).code === "ABORT_ERR") return true;
return false;
}
function retryPlanDetails(
plan: RetryPlan,
opts: HttpPostOpts,
maxRetries: number,
status: number | null,
attempt: number,
): Record {
return {
provider: opts.provider,
url: opts.url,
status,
attempt,
maxRetries,
backoffMs: plan.backoffMs,
plannedDelayMs: plan.delayMs,
retryAfterMs: plan.retryAfterMs,
retryAt: plan.retryAt,
retrySource: plan.source,
retryDecision: plan.action,
...(plan.action === "defer" ? { retryReason: plan.reason } : {}),
remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt),
};
}
function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null {
return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs);
}
function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal {
if (!a) return b;
const ctrl = new AbortController();
const forward = () => ctrl.abort();
if (a.aborted || b.aborted) ctrl.abort();
a.addEventListener("abort", forward, { once: true });
b.addEventListener("abort", forward, { once: true });
return ctrl.signal;
}
function serializeErr(e: unknown): Record {
if (e instanceof Error) {
return { name: e.name, message: e.message };
}
return { value: String(e) };
}