import { Impit, type ImpitResponse } from "impit"; import { parseFetchedContent, type FetchContentKind } from "./fetch-formats.ts"; import { type DnsLookup, validatePublicHttpUrl } from "./ssrf.ts"; export const DEFAULT_FETCH_TIMEOUT_MS = 30_000; export const DEFAULT_FETCH_MAX_BYTES = 10 * 1024 * 1024; export const DEFAULT_FETCH_MAX_REDIRECTS = 5; const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); export interface ImpitLike { fetch(resource: string | URL | Request, init?: { method?: "GET"; headers?: Headers | Record | [string, string][]; timeout?: number; signal?: AbortSignal; redirect?: "manual"; }): Promise; } export interface ImpitResponseLike { status: number; statusText: string; headers: Headers; ok: boolean; url: string; body: ReadableStream; abort?: () => void; } export interface WebFetchSnapshot { requestedUrl: string; finalUrl: string; status: number; statusText: string; headers: Record; redirects: string[]; contentType: string; kind: FetchContentKind; sourceFormat: string; extension: string; charset: string; rawBody: Uint8Array; title: string; sourceText?: string; text?: string; markdown?: string; parseWarning?: string; } export interface ImpitFetchOptions { signal?: AbortSignal; timeoutMs?: number; maxBytes?: number; maxRedirects?: number; lookup?: DnsLookup; client?: ImpitLike; validateUrl?: (url: string | URL) => Promise; } function responseHeaders(headers: Headers): Record { const result: Record = {}; headers.forEach((value, key) => { result[key] = value; }); return result; } function parseContentType(value: string | null): { mediaType: string; charset: string } { const [rawMediaType = "", ...parameters] = (value ?? "").split(";"); const mediaType = rawMediaType.trim().toLowerCase(); let charset = "utf-8"; for (const parameter of parameters) { const match = parameter.match(/^\s*charset\s*=\s*["']?([^"';\s]+)["']?\s*$/i); if (match?.[1]) charset = match[1].toLowerCase(); } return { mediaType, charset }; } async function readLimitedBody(response: ImpitResponseLike, maxBytes: number): Promise { const declared = response.headers.get("content-length"); if (declared && Number(declared) > maxBytes) { response.abort?.(); throw new Error(`web_fetch response exceeds ${maxBytes} byte limit (Content-Length: ${declared})`); } const reader = response.body.getReader(); const chunks: Uint8Array[] = []; let total = 0; try { while (true) { const item = await reader.read(); if (item.done) break; const chunk = item.value; total += chunk.byteLength; if (total > maxBytes) { response.abort?.(); await reader.cancel().catch(() => {}); throw new Error(`web_fetch response exceeds ${maxBytes} byte limit`); } chunks.push(chunk); } } finally { reader.releaseLock(); } const body = new Uint8Array(total); let offset = 0; for (const chunk of chunks) { body.set(chunk, offset); offset += chunk.byteLength; } return body; } function combineSignal(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { const timeout = AbortSignal.timeout(timeoutMs); return signal ? AbortSignal.any([signal, timeout]) : timeout; } function makeImpit(timeoutMs: number): ImpitLike { return new Impit({ browser: "chrome", ignoreTlsErrors: false, vanillaFallback: false, timeout: timeoutMs, followRedirects: false, maxRedirects: 0, }) as unknown as ImpitLike; } export async function fetchWithImpit(url: string, options: ImpitFetchOptions = {}): Promise { const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS; const maxBytes = options.maxBytes ?? DEFAULT_FETCH_MAX_BYTES; const maxRedirects = options.maxRedirects ?? DEFAULT_FETCH_MAX_REDIRECTS; const signal = combineSignal(options.signal, timeoutMs); const client = options.client ?? makeImpit(timeoutMs); const validate = options.validateUrl ?? ((value: string | URL) => validatePublicHttpUrl(value, options.lookup)); const requested = await validate(url); let current = requested; const redirects: string[] = []; for (let redirectCount = 0; redirectCount <= maxRedirects; redirectCount += 1) { if (signal.aborted) throw signal.reason instanceof Error ? signal.reason : new Error("web_fetch aborted"); const response = await client.fetch(current, { method: "GET", redirect: "manual", timeout: timeoutMs, signal, headers: { accept: "text/html,application/xhtml+xml,text/markdown,text/plain;q=0.9,*/*;q=0.1", "accept-language": "en-US,en;q=0.9", "cache-control": "no-cache", }, }); if (REDIRECT_STATUSES.has(response.status)) { const location = response.headers.get("location"); response.abort?.(); if (!location) throw new Error(`HTTP ${response.status} redirect has no Location header`); if (redirectCount === maxRedirects) throw new Error(`web_fetch exceeded ${maxRedirects} redirects`); current = await validate(new URL(location, current)); redirects.push(current.toString()); continue; } if (!response.ok) { response.abort?.(); throw new Error(`web_fetch failed with HTTP ${response.status}: ${response.statusText}`); } const rawBody = await readLimitedBody(response, maxBytes); const parsedType = parseContentType(response.headers.get("content-type")); const finalUrl = response.url || current.toString(); const parsed = await parseFetchedContent(rawBody, { mediaType: parsedType.mediaType, declaredCharset: parsedType.charset, finalUrl, contentDisposition: response.headers.get("content-disposition") ?? undefined, signal, }); return { requestedUrl: requested.toString(), finalUrl, status: response.status, statusText: response.statusText, headers: responseHeaders(response.headers), redirects, contentType: parsedType.mediaType || "application/octet-stream", rawBody, ...parsed, }; } throw new Error(`web_fetch exceeded ${maxRedirects} redirects`); } export type { ImpitResponse };