/**
 * HTTP transport layer — undici Agent with caching and retry.
 *
 * Provides `fetchUrl()` with:
 *  - ETag-based conditional requests (304 → cached body)
 *  - Grant-based caching: a response is stored only when the server granted
 *    freshness — a time grant (`Cache-Control: max-age`) or a validator grant
 *    (an ETag → revalidate-only; the body is stored and every repeat is a
 *    conditional GET, usually answered by a 304 with no body). No TTL is
 *    ever fabricated for header-less responses.
 *  - Retry-on-429 with exponential backoff + `Retry-After` support
 *  - Configurable timeout
 *  - Automatic charset decoding from Content-Type
 *
 * Caching is per-URL, module-level (in-memory map). The cache is
 * transparent to callers; each URL is cached independently.
 *
 * Deliberately ignored cache directives (recorded so they aren't
 * re-investigated):
 *  - `Vary` — known limitation: the cache keys on URL + Accept + charset
 *    only; pre-existing, not widened into a fix here.
 *  - `s-maxage` — correctly ignored (private cache, RFC 7234 §1.3).
 *  - `must-revalidate` / `stale-while-revalidate` / `stale-if-error` —
 *    trivially satisfied: nothing is ever served stale.
 *  - `Age` / `Date` skew — a granted entry is served for its full `max-age`
 *    from store time even if `Age` says it arrived stale; pre-existing,
 *    unchanged in severity.
 *  - `Expires` — parsing deliberately not implemented (HTTP/1.0 legacy);
 *    under grant-based caching an Expires-only server is either
 *    revalidate-only (ETag present) or refetched every time (none).
 */

import { readFileSync } from "node:fs";
import { request, Agent, interceptors, type Dispatcher } from "undici";
import { gunzipSync, inflateRawSync, inflateSync } from "node:zlib";
import { ssrfGuard } from "./ssrf-guard.js";

// User-Agent version tracks the package version (no release-script edit needed).
// Falls back to "unknown" if package.json is unreadable — never fail transport init.
let HOST_UA_VERSION = "unknown";
try {
	HOST_UA_VERSION = (
		JSON.parse(
			readFileSync(new URL("../package.json", import.meta.url), "utf-8"),
		) as { version: string }
	).version;
} catch {
	// keep fallback
}

// ponytail: module-level composed agents; closed only on process exit.
// Add a close() in session_shutdown if leak-detection ever flags it.

/** One-shot flag so a 1-char path secret warns once, not per redacted URL. */
let warnedOneCharPathSecret = false;

/** Auto-follows up to 5 redirects. Used when redirect targets are trusted
 *  (agent-supplied URLs — the agent has bash, so guarding them is theater). */
const redirectAgent = new Agent().compose(
	interceptors.redirect({ maxRedirections: 5 }),
);

/** No auto-redirect. Used with `guardRedirects` so each redirect target is
 *  SSRF-checked before it is followed. See fetchUrl for the manual loop. */
const noRedirectAgent = new Agent();

const MAX_REDIRECTS = 5;

// ═══════════════════════════════════════════════════════════════════
// Types
// ═══════════════════════════════════════════════════════════════════

export interface FetchOptions {
	/** Request timeout in ms (default: 30_000). */
	timeout?: number;
	/** Additional request headers. */
	headers?: Record<string, string>;
	/** Max retries on 429 (default: 2). */
	maxRetries?: number;
	/** Skip cache (force fresh fetch). */
	fresh?: boolean;
	/** SSRF-check each redirect target before following it. Use it for
	 *  server-supplied URLs (paginate nextLink) — a malicious API can 302
	 *  to an internal host, and when auth headers ship the Authorization
	 *  header would attach to the redirect. Agent-supplied URLs don't need
	 *  this (the agent already has bash). Forced on automatically for any
	 *  auth-bearing request (hasAuth), so a keyed call is always guarded. */
	guardRedirects?: boolean;
	/**
	 * Lowercased header names whose values are secrets injected from the
	 * secrets store (kind: static-key). Stripped — along with `authorization`
	 * — on cross-domain redirect hops so a secret can't leak to another host.
	 * Only honored while the guarded-redirect path is active (forced for any
	 * auth-bearing request, so this always applies to keyed calls).
	 */
	secretHeaderNames?: Set<string>;
	/**
	 * Query-param names whose values are secrets injected from the secrets
	 * store (static-key `secretQueryRefs` or oauth2 `paramStyle: query`).
	 * Stripped from the URL on cross-domain redirect hops — a relative
	 * redirect preserves the original query, so a secret param would
	 * otherwise ride the hop to another host. Only honored while the
	 * guarded-redirect path is active (same condition as secretHeaderNames).
	 */
	secretQueryParamNames?: Set<string>;
	/**
	 * True when this request carries a store-injected query-param secret
	 * (kind: static-key). Broadens `hasAuth` to cover query-secret-only
	 * guides (header-secret callers never set this): the response becomes
	 * private (never cached / served from cache) and redirects are guarded.
	 */
	hasQuerySecret?: boolean;
	/**
	 * True when this request carries store-injected path-token secrets
	 * (kind: static-key `secretPathRefs`). Broadens `hasAuth` exactly like
	 * `hasQuerySecret` — a path-secret-only guide (Telegram-class: no secret
	 * header, no secret query param) must skip the cache and get guarded
	 * redirects.
	 */
	hasPathSecret?: boolean;
	/**
	 * Redaction closure applied to every cross-domain redirect hop URL before
	 * the next fetch: replaces store-injected path-token values (raw + both
	 * hex-encoded forms) with `***`. A path token rides mid-segment
	 * (`/bot<token>/getUpdates`), so a relative redirect would carry it to the
	 * other host — this is a value replacement, not a segment deletion (that
	 * would change the route). Built by the caller where the values are in
	 * scope; the transport stays value-agnostic (no raw secrets in its
	 * signature vocabulary). Only consulted on cross-domain hops — same-host
	 * Location echoes keep the token (resolve-against-hopUrl semantics).
	 */
	redactPathSecret?: (url: string) => string;
	/** Charset to decode the body with when the response's Content-Type
	 *  header omits one. Honors a guide's `responseShape.charset` for APIs
	 *  that serve e.g. ISO-8859-1 bytes without a charset parameter. An
	 *  explicit header charset always takes precedence. */
	fallbackCharset?: string;
}

export interface FetchResult {
	status: number;
	headers: Record<string, string>;
	body: string;
	/** True when the result came from cache (no network request). */
	cached: boolean;
	/** Final URL after redirects — present only when at least one hop occurred. */
	finalUrl?: string;
}

// ═══════════════════════════════════════════════════════════════════
// Cache
// ═══════════════════════════════════════════════════════════════════

interface CacheEntry {
	body: string;
	/** ETag for conditional requests; absent when the upstream didn't send one. */
	etag?: string;
	expiresAt: number;
	/** The freshness grant (ms) currently in effect — stamped at store time
	 *  and re-stamped on every 304 refresh. A bare 304 (no grant of its own)
	 *  falls back to this value so a granted entry doesn't degrade to
	 *  revalidate-only when the 304 carries no max-age; because it is
	 *  re-stamped, a `no-cache` 304's 0 grant survives later bare 304s. */
	grantMs: number;
}

const DEFAULT_TIMEOUT = 30_000;
const DEFAULT_MAX_RETRIES = 2;
const MAX_CACHE_ENTRIES = 100; // ponytail: hard cap; evict soonest-expiring when exceeded
const MAX_BODY_BYTES = 10 * 1024 * 1024; // 10 MB ceiling on a single response body

/** Module-level cache: cache-key → CacheEntry. */
const cache = new Map<string, CacheEntry>();

/** Module clock — overridden by tests for sleep-free TTL-expiry scenarios.
 *  Only the four cache-path call sites consult this (hit branch,
 *  evictCacheIfNeeded, 2xx store, 304 refresh); retry/backoff/timeout paths
 *  keep bare Date.now(). */
let _now: () => number = Date.now;

/** Test seam — swap the module clock used by the cache paths.
 *  Same `_…ForTest` convention as guide-store.ts's `_resetLoadWarningsForTest`. */
export function _setClockForTest(fn: () => number): void {
	_now = fn;
}

/**
 * Drop expired entries, then — if still over the cap — evict the
 * soonest-expiring entries. Called before every `cache.set`.
 */
function evictCacheIfNeeded(): void {
	if (cache.size < MAX_CACHE_ENTRIES) return;
	const now = _now();
	for (const [k, e] of cache) {
		if (e.expiresAt <= now) cache.delete(k);
	}
	if (cache.size >= MAX_CACHE_ENTRIES) {
		const sorted = [...cache.entries()].sort(
			(a, b) => a[1].expiresAt - b[1].expiresAt,
		);
		while (cache.size >= MAX_CACHE_ENTRIES && sorted.length > 0) {
			const next = sorted.shift();
			if (next) cache.delete(next[0]);
		}
	}
}

// ═══════════════════════════════════════════════════════════════════
// Internal helpers
// ═══════════════════════════════════════════════════════════════════

/**
 * Produce a cache key from URL and optional Accept header.
 *
 * Two requests for the same URL with different Accept values are cached
 * separately so that content-negotiated responses don't collide.
 * When no Accept header is present the plain URL is used as the key.
 */
function cacheKey(url: string, opts?: FetchOptions): string {
	const accept = opts?.headers?.accept;
	const cs = opts?.fallbackCharset ?? "";
	const suffix =
		(accept ? `\x00accept=${accept}` : "") + (cs ? `\x00cs=${cs}` : "");
	return suffix ? `${url}${suffix}` : url;
}

/**
 * Output-channel audit — URL channel: redact every secret query param's
 * value to `***` in a URL for surfacing. Returns the URL unchanged when no
 * secret param names are in play (so non-secret guides never get URL-
 * normalized by this). Defined here so the transport layer — the one place
 * that holds a raw request URL and may embed it in an error message — can
 * self-redact instead of relying on callers to remember. `helpers.ts`
 * re-exports it for the capture points it owns (result.url, urls[],
 * HelperError.url).
 */
export function redactSecretParams(
	url: string,
	secretParamNames?: Set<string>,
): string {
	if (!secretParamNames || secretParamNames.size === 0) return url;
	try {
		const u = new URL(url);
		for (const name of secretParamNames) {
			if (u.searchParams.has(name)) u.searchParams.set(name, "***");
		}
		return u.toString();
	} catch {
		return url;
	}
}

/**
 * Output-channel audit — path-token channel: redact store-injected path
 * token values from a URL for surfacing. Unlike `redactSecretParams` (query
 * params, name-keyed), a path token is embedded mid-segment
 * (`/bot<token>/getUpdates`), so this is a string replace of each resolved
 * value with `***` — raw, plus its `encodeURIComponent` form in BOTH hex
 * cases (uppercase `%3A` from `fillPathTemplate` and lowercase `%3a` echoes
 * from server-normalized URLs / error bodies). Deliberate asymmetry with
 * `scrubSecretValues` (do not unify): the URL channel skips 1-char values
 * because surfaced URLs are functional — a 1-char replace corrupts unrelated
 * path text (`/things/1234/x` → `/things/***234/x`) with negligible security
 * gain — while the body scrub takes any non-empty value (bodies are
 * informational and tolerate collateral corruption). String needles only,
 * never regex — same `$`-pattern pitfall `scrubSecretValues` avoids.
 */
export function redactSecretPathValues(
	url: string,
	secretPathValues?: string[],
): string {
	if (!secretPathValues || secretPathValues.length === 0) return url;
	let out = url;
	for (const v of secretPathValues) {
		if (v.length === 0) continue;
		if (v.length === 1) {
			// ponytail: once-per-session flag — a long /api verify run would
			// otherwise spam one warn per redacted URL; promote to a per-guide
			// (or surfaced-details) channel if the once-per-session miss matters.
			if (!warnedOneCharPathSecret) {
				warnedOneCharPathSecret = true;
				console.warn(
					"⚠ path secret is 1 character — redact-unfriendly for URL surfaces; skipping URL redaction (the body scrub still applies)",
				);
			}
			continue;
		}
		const enc = encodeURIComponent(v);
		const encLower = enc.replace(/%../g, (s) => s.toLowerCase());
		for (const needle of new Set([v, enc, encLower])) {
			out = out.split(needle).join("***");
		}
	}
	return out;
}

function parseHeaders(
	hdrs: Dispatcher.ResponseData["headers"],
): Record<string, string> {
	const out: Record<string, string> = {};
	if (hdrs && typeof hdrs === "object") {
		// undici returns headers as an object with lowercased keys.
		for (const [key, val] of Object.entries(hdrs)) {
			if (typeof val === "string") {
				out[key.toLowerCase()] = val;
			} else if (Array.isArray(val)) {
				// Repeated headers arrive as string[] — join like raw form.
				out[key.toLowerCase()] = val.join(", ");
			}
		}
	}
	return out;
}

async function collectBody(
	body: Dispatcher.ResponseData["body"],
): Promise<Buffer> {
	const chunks: Buffer[] = [];
	let total = 0;
	for await (const chunk of body) {
		total += chunk.length;
		if (total > MAX_BODY_BYTES) {
			const err = new Error(`Response body exceeded ${MAX_BODY_BYTES} bytes`);
			err.name = "BodyTooLargeError"; // checked by fetchUrl retry logic
			throw err;
		}
		chunks.push(Buffer.from(chunk));
	}
	return Buffer.concat(chunks);
}

/**
 * Content-Encoding decompression — some servers (Open Food Facts' mod_deflate,
 * various OData stacks) gzip JSON responses even when the client does not
 * advertise `Accept-Encoding: gzip`. undici's raw `request()` does NOT
 * auto-decompress (unlike fetch()), so without this the raw gzip bytes reach
 * JSON.parse and the op fails with a confusing "Invalid JSON response".
 * Handled here — the single choke point every fetch (restGet, paginate,
 * nextLink hops, /api verify) flows through — not per call site.
 * ponytail: gzip + deflate only; br and multi-layer encodings pass through
 * untouched — add when a real guide needs them.
 */
function decompressBody(buf: Buffer, encoding: string | undefined): Buffer {
	const enc = encoding?.toLowerCase().trim();
	if (!enc) return buf;
	try {
		if (enc === "gzip" || enc === "x-gzip") return gunzipSync(buf);
		// "deflate" is ambiguous on the wire (zlib-wrapped vs raw) — try both.
		if (enc === "deflate") {
			try {
				return inflateSync(buf);
			} catch {
				return inflateRawSync(buf);
			}
		}
	} catch {
		// Un-decodable body (e.g. server lied about the encoding) — return the
		// raw bytes so the caller surfaces the parse error it would have anyway.
	}
	return buf;
}

function decodeBuffer(buf: Buffer, charset: string): string {
	const cs = charset.toLowerCase().replace(/[^a-z0-9_-]/g, "");
	if (cs === "utf-8" || cs === "utf8" || cs === "") {
		return buf.toString("utf-8");
	}
	try {
		const decoder = new TextDecoder(cs);
		return decoder.decode(buf);
	} catch {
		// Fallback to utf-8 if TextDecoder doesn't recognise the charset.
		return buf.toString("utf-8");
	}
}

function parseMaxAge(headers: Record<string, string>): number | null {
	const cc = headers["cache-control"];
	if (!cc) return null;
	const m = cc.match(/max-age=(\d+)/i);
	return m ? parseInt(m[1]!, 10) * 1000 : null;
}

/**
 * Effective freshness grant for a response, in ms — computed ONCE per
 * response; every cache consumer (store condition, entry expiry, 304
 * refresh) reads this single derivation, nothing re-derives from raw
 * headers.
 *
 *   no-cache  → 0  (revalidation required before reuse regardless of
 *                  freshness, RFC 7234 §5.2.2.4 — even with max-age > 0)
 *   otherwise → parseMaxAge(headers) ?? 0  (0 = no time grant; a
 *                  revalidate-only entry is born expired)
 *
 * `no-store` is NOT detected here — the store/refresh sites check it
 * explicitly, because a `no-store` response carrying an ETag (or max-age)
 * would otherwise satisfy the grant formula and get stored.
 */
function effectiveGrant(headers: Record<string, string>): number | null {
	const cc = headers["cache-control"];
	if (cc && /no-cache/i.test(cc)) return 0;
	return parseMaxAge(headers);
}

/**
 * Cache-store condition for a 2xx response: cache the entry only when the
 * response is not `no-store` AND (the effective grant is > 0 or an ETag
 * exists). Pass the grant already derived by the caller — the same
 * "compute once" rule the store/expiry/304 consumers follow.
 * `grantMs > 0` — not raw `parseMaxAge` — excludes dead entries
 * (`max-age=0` or `no-cache, max-age>0` with no ETag would be stored,
 * never served, never 304-refreshable); `no-cache` is not excluded from
 * storage outright because with an ETag it revalidates every use (RFC
 * 7234), consistent with the `no-cache, max-age=0` + ETag pair
 * (`parseMaxAge` returns 0, not null, for that pair).
 */
function shouldStore(
	headers: Record<string, string>,
	grantMs: number,
): boolean {
	if (/no-store/i.test(headers["cache-control"] ?? "")) return false;
	return grantMs > 0 || !!headers["etag"];
}

/**
 * Compute the delay before a retry. Prefers Retry-After: delay-seconds;
 * falls back to parsing an HTTP-date form. If the date is in the past
 * (clock skew / already-expired) or absent, uses exponential backoff —
 * never a negative or zero delay.
 */
export function waitForRetry(
	headers: Record<string, string>,
	attempt: number,
): number {
	const raw = headers["retry-after"];
	if (raw) {
		const secs = parseInt(raw, 10);
		if (!Number.isNaN(secs) && secs > 0) return secs * 1000;
		const dateMs = Date.parse(raw);
		if (!Number.isNaN(dateMs)) {
			const delta = dateMs - Date.now();
			if (delta > 0) return delta;
		}
	}
	// Exponential backoff: 1s, 2s, 4s, …
	return Math.min(1000 * 2 ** attempt, 30_000);
}

/**
 * One GET request with its own abort timer. Returns status, parsed headers,
 * and the raw collected body. Redirects are NOT followed here — the caller
 * decides whether to follow and guard them.
 */
async function singleGet(
	url: string,
	reqHeaders: Record<string, string>,
	timeoutMs: number,
	dispatcher: Dispatcher,
): Promise<{
	status: number;
	headers: Record<string, string>;
	rawBody: Buffer;
	finalUrl: string;
}> {
	if (timeoutMs <= 0) throw new Error("Request timeout");
	const ac = new AbortController();
	const timer = setTimeout(
		() => ac.abort(new Error("Request timeout")),
		timeoutMs,
	);
	try {
		const resp = await request(url, {
			method: "GET",
			headers: reqHeaders,
			signal: ac.signal,
			dispatcher,
		});
		// The redirect interceptor records the hop chain in context.history;
		// its last entry is where the response actually came from. No history
		// (no interceptor / no redirect) means the URL is unchanged.
		const history = (resp.context as { history?: URL[] } | null)?.history;
		return {
			status: resp.statusCode,
			headers: parseHeaders(resp.headers),
			rawBody: await collectBody(resp.body),
			finalUrl: history?.at(-1) ? String(history.at(-1)) : url,
		};
	} finally {
		clearTimeout(timer);
	}
}

/** Host of a URL, or null when unparseable — treated as cross-domain (strip). */
function hostOf(url: string): string | null {
	try {
		return new URL(url).host;
	} catch {
		return null;
	}
}

/**
 * Drop store-injected secret headers (plus `authorization`) from a request
 * before it leaves the request's original host. Literal auth.headers (not in
 * secretHeaderNames) survive. Used on cross-domain redirect hops only.
 * Exported for the output-channel/SSRF structural tests (case c: cross-domain).
 */
export function stripSecretHeaders(
	headers: Record<string, string>,
	secretHeaderNames?: Set<string>,
): Record<string, string> {
	const drop = new Set<string>(["authorization"]);
	if (secretHeaderNames)
		for (const h of secretHeaderNames) drop.add(h.toLowerCase());
	const out: Record<string, string> = {};
	for (const [k, v] of Object.entries(headers)) {
		if (drop.has(k.toLowerCase())) continue;
		out[k] = v;
	}
	return out;
}

/**
 * Remove secret query params from a URL (cross-domain redirect hops only).
 * A relative redirect (new URL("?x", current)) preserves the original query
 * string, so injected secrets would survive onto the next host.
 */
export function stripSecretQueryParams(
	url: string,
	secretQueryParamNames?: Set<string>,
): string {
	if (!secretQueryParamNames || secretQueryParamNames.size === 0) return url;
	// ponytail: URL() throws on malformed input; pass through unmodified —
	// singleGet will surface the real fetch error anyway.
	let u: URL;
	try {
		u = new URL(url);
	} catch {
		return url;
	}
	let changed = false;
	for (const name of secretQueryParamNames) {
		if (u.searchParams.has(name)) {
			u.searchParams.delete(name);
			changed = true;
		}
	}
	return changed ? u.toString() : url;
}

/**
 * Follow redirects manually, SSRF-checking each target. Used for
 * server-supplied URLs (paginate nextLink) and — forced — any auth-bearing
 * request. GET-only, so method is preserved across 301/302/303/307/308
 * trivially. Returns the final response; a redirect to a blocked host
 * throws before it is fetched. Store-injected secrets are stripped on
 * cross-domain hops so a key can't attach to a different host.
 */
async function getWithGuardedRedirects(
	url: string,
	reqHeaders: Record<string, string>,
	startTime: number,
	timeoutMs: number,
	secretHeaderNames?: Set<string>,
	secretQueryParamNames?: Set<string>,
	redactPathSecret?: (url: string) => string,
): Promise<{
	status: number;
	headers: Record<string, string>;
	rawBody: Buffer;
	finalUrl: string;
}> {
	let current = url;
	const originalHost = hostOf(url);
	for (let hops = 0; hops <= MAX_REDIRECTS; hops++) {
		// Strip store-injected secrets (and Authorization) once a hop leaves
		// the request's original host — a secret must never cross domains.
		// Query secrets need the same treatment: a relative redirect preserves
		// the original query string, so secret query params would ride the hop.
		// Path tokens ride mid-segment, so the hop URL additionally gets the
		// caller's redaction closure (value replace, never segment deletion).
		const crossDomain = hostOf(current) !== originalHost;
		const hopHeaders = crossDomain
			? stripSecretHeaders(reqHeaders, secretHeaderNames)
			: reqHeaders;
		let hopUrl = current;
		if (crossDomain) {
			hopUrl = stripSecretQueryParams(current, secretQueryParamNames);
			if (redactPathSecret) hopUrl = redactPathSecret(hopUrl);
		}
		const remaining = timeoutMs - (Date.now() - startTime);
		const res = await singleGet(hopUrl, hopHeaders, remaining, noRedirectAgent);
		const isRedirect =
			res.status >= 300 && res.status < 400 && res.status !== 304;
		if (!isRedirect || hops === MAX_REDIRECTS) return res;
		const loc = res.headers["location"];
		if (!loc) return res; // redirect with no Location — return as-is
		// Resolve against hopUrl, not current — a relative redirect inherits
		// the base's query string, and current still carries the secret params
		// that were just stripped for this hop's fetch.
		const next = new URL(loc, hopUrl).toString();
		const guard = ssrfGuard(next);
		if (!guard.ok) {
			// Security-control failure is not transient — don't retry.
			const err = new Error(`Redirect to blocked host: ${guard.reason}`);
			err.name = "SsrfBlockedError";
			throw err;
		}
		current = next;
	}
	// Unreachable: the loop returns on the MAX_REDIRECTS hop.
	throw new Error("Too many redirects");
}

// ═══════════════════════════════════════════════════════════════════
// Public API
// ═══════════════════════════════════════════════════════════════════

/**
 * Fetch a URL with caching and retry-on-429.
 *
 * Cache behaviour (grant-based — no TTL is ever fabricated):
 *  - URLs are cached in a module-level `Map<string, CacheEntry>`.
 *  - A 2xx is stored only on an explicit server grant: a time grant
 *    (`Cache-Control: max-age=N`) or a validator grant (an ETag — stored
 *    with no stale window; every repeat sends `If-None-Match` and is
 *    revalidate-only). `no-store` is never stored; `no-cache` never
 *    serves from the fresh-hit branch (with an ETag it still revalidates).
 *  - When a cached entry has an `etag`, the conditional `If-None-Match`
 *    header is sent; a 304 refreshes the entry's expiry from the 304's
 *    own grant, falling back to the grant recorded at store time. A 304
 *    carrying `no-store` deletes the entry instead of refreshing it.
 *  - Header-less responses (no `Cache-Control`, no ETag) are never
 *    cached — every call hits the network.
 *  - `opts.fresh = true` skips the cache read and bypasses sending
 *    `If-None-Match` (server returns full response); a granted response
 *    fetched fresh is still stored (seeding the cache).
 *  - Requests carrying caller-specific headers (anything besides Accept,
 *    e.g. API keys) are never cached or served from cache — the response
 *    is private to that caller.
 *
 * Retry behaviour:
 *  - 429 responses are retried up to `opts.maxRetries` times.
 *  - Backoff: if `Retry-After` is present, wait that many seconds;
 *    otherwise exponential backoff (1s, 2s, 4s, capped at 30s).
 *  - Other 4xx/5xx status codes are returned as-is without retry.
 *
 * Network errors (DNS, connection refused, timeout) are thrown.
 */
export async function fetchUrl(
	url: string,
	opts?: FetchOptions,
): Promise<FetchResult> {
	const timeout = opts?.timeout ?? DEFAULT_TIMEOUT;
	const maxRetries = opts?.maxRetries ?? DEFAULT_MAX_RETRIES;

	// Responses to requests carrying caller-specific headers (auth keys,
	// tokens, …) are private to that caller — never cache or reuse them.
	// Only the Accept header (content negotiation) is cache-shareable, and
	// it's already part of the cache key. Without this gate, the same URL
	// fetched with two different auth headers would collide on one cache
	// entry and the second caller would get the first caller's response.
	const hasAuthHeaders =
		!!opts?.headers &&
		Object.keys(opts.headers).some((h) => h.toLowerCase() !== "accept");
	// Broader hasAuth gate: header-secrets ∨ query-secrets ∨ path-secrets. A
	// query- or path-secret-only guide carries no non-accept header, so
	// hasAuthHeaders alone would miss it — hasQuerySecret/hasPathSecret close
	// those gaps. Every auth gate below keys on hasAuth: cache-skip,
	// If-None-Match, and the guarded-redirect force.
	const hasAuth =
		hasAuthHeaders ||
		(opts?.hasQuerySecret ?? false) ||
		(opts?.hasPathSecret ?? false);

	// ── cache hit ───────────────────────────────────────────────
	const key = cacheKey(url, opts);
	if (!opts?.fresh && !hasAuth) {
		const entry = cache.get(key);
		if (entry && _now() < entry.expiresAt) {
			return { status: 200, headers: {}, body: entry.body, cached: true };
		}
	}

	// ── request headers ─────────────────────────────────────────
	const reqHeaders: Record<string, string> = { ...opts?.headers };
	if (!reqHeaders["user-agent"]) {
		reqHeaders["user-agent"] =
			`pi-lean-host/${HOST_UA_VERSION} (+https://github.com/coreyryanhanson/pi-lean-dimension)`;
	}

	// The entry being revalidated, captured at If-None-Match time. The 304
	// arm reads this instead of re-fetching the map: a concurrent caller's
	// evictCacheIfNeeded can delete the (expired) entry while the conditional
	// GET is awaited, and re-reading would then find nothing and serve the
	// 304's empty body as the response.
	let revalidate: CacheEntry | undefined;

	if (!opts?.fresh && !hasAuth) {
		const entry = cache.get(key);
		if (entry?.etag) {
			reqHeaders["If-None-Match"] = entry.etag;
			revalidate = entry;
		}
	}

	const startTime = Date.now();

	for (let attempt = 0; ; attempt++) {
		const remaining = timeout - (Date.now() - startTime);
		if (remaining <= 0) throw new Error("Request timeout");

		try {
			// Guarded redirects when explicitly requested (server-supplied
			// nextLink) OR when the request is auth-bearing (hasAuth) — a keyed
			// call is always SSRF-checked hop-by-hop and its secrets stripped on
			// cross-domain redirects. Otherwise let undici auto-follow up to 5.
			const useGuarded = (opts?.guardRedirects ?? false) || hasAuth;
			const {
				status,
				headers: respHeaders,
				rawBody,
				finalUrl,
			} = useGuarded
				? await getWithGuardedRedirects(
						url,
						reqHeaders,
						startTime,
						timeout,
						opts?.secretHeaderNames,
						opts?.secretQueryParamNames,
						opts?.redactPathSecret,
					)
				: await singleGet(url, reqHeaders, remaining, redirectAgent);

			// ── 304 Not Modified ────────────────────────────────
			// Gated on !hasAuth like every other cache arm: a keyed request must
			// never be served a body cached by an unauthenticated caller, even
			// from a protocol-violating server that sends a spontaneous 304.
			// Also gated on !fresh for the same reason: a fresh request must not
			// receive a cached body, so an unsolicited 304 falls through to the
			// status check instead.
			if (status === 304 && !hasAuth && !opts?.fresh) {
				const entry = revalidate ?? cache.get(key);
				if (entry) {
					if (/no-store/i.test(respHeaders["cache-control"] ?? "")) {
						// A no-store revalidation result must not be stored.
						cache.delete(key);
					} else {
						// Refresh from the 304's own effective grant; fall back to the
						// grant currently recorded on the entry (a granted entry must
						// not degrade to revalidate-only when the 304 carries no
						// max-age), and re-stamp it so the newest grant sticks.
						const grant304 = effectiveGrant(respHeaders) ?? entry.grantMs;
						entry.grantMs = grant304;
						entry.expiresAt = _now() + grant304;
						// The entry object is mutated in place — cache.get returned the
						// live reference, so no re-set is needed.
					}
					return {
						status: 200,
						headers: respHeaders,
						body: entry.body,
						cached: true,
						...(finalUrl === url ? {} : { finalUrl }),
					};
				}
				// No cached entry → fall through to process body.
			}

			// ── 429 Too Many Requests ───────────────────────────
			if (status === 429 && attempt < maxRetries) {
				const delay = waitForRetry(respHeaders, attempt);
				await new Promise((r) => setTimeout(r, delay));
				continue; // retry
			}

			// ── decode & cache (2xx only) ───────────────────────
			const contentType = respHeaders["content-type"] ?? "";
			const charsetMatch = contentType.match(/charset\s*=\s*([^\s;]+)/i);
			const charset = charsetMatch?.[1] ?? opts?.fallbackCharset ?? "utf-8";
			const body = decodeBuffer(
				decompressBody(rawBody, respHeaders["content-encoding"]),
				charset,
			);

			if (status >= 200 && status < 300 && !hasAuth) {
				const grantMs = effectiveGrant(respHeaders) ?? 0;
				if (shouldStore(respHeaders, grantMs)) {
					const entry: CacheEntry = {
						body,
						expiresAt: _now() + grantMs,
						grantMs,
					};
					const etag = respHeaders["etag"];
					if (etag) entry.etag = etag;
					evictCacheIfNeeded();
					cache.set(key, entry);
				}
			}

			return {
				status,
				headers: respHeaders,
				body,
				cached: false,
				...(finalUrl === url ? {} : { finalUrl }),
			};
		} catch (err) {
			const e = err instanceof Error ? err : new Error(String(err));
			// Don't retry on timeout/abort, oversized bodies, or SSRF
			// blocks — none are transient.
			const transient =
				e.name !== "AbortError" &&
				e.name !== "BodyTooLargeError" &&
				e.name !== "SsrfBlockedError" &&
				(e as NodeJS.ErrnoException).code !== "UND_ERR_ABORTED";
			if (attempt < maxRetries && transient) {
				const delay = Math.min(1000 * 2 ** attempt, 30_000);
				await new Promise((r) => setTimeout(r, delay));
				continue;
			}
			throw e;
		}
	}
}