/** @file Http fingerprint-adapter module. */ import { isIP } from "node:net"; import { DEFAULT_MAX_BYTES, DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT } from "../../defaults.ts"; import type { FetchUrlResult, HttpClientOptions } from "../client.ts"; import { HttpClient } from "../client.ts"; import { normalizeHeaders } from "../download.ts"; import { HttpClientError, httpClientErrorFromUnknown } from "../errors.ts"; import { PolitenessController } from "../politeness.ts"; import { resolveEnvProxyForUrl } from "../proxy-config.ts"; import { isSocksProxyUrl, validateProxyUrl } from "../proxy-dispatcher.ts"; import { followRedirects } from "../redirects.ts"; import { fetchWithRequestPolicy } from "../request-policy.ts"; import { materializeFetchBufferResponse, materializeFetchStreamResponse } from "../response.ts"; import { loadRobotsText, RobotsCache } from "../robots.ts"; import { getOrCreateSession, mergeSessionHeaders, updateSessionCookies, type FetchSession, } from "../session.ts"; import { withTimeout } from "../timeout.ts"; import { assertSafeFetchUrl, UrlSafetyError, type SafeUrlResult } from "../url-safety.ts"; import { assertSupportedFingerprintOptions, type FingerprintBackendFactory, type FingerprintBackendResponse, type FingerprintFetchAdapter, type FingerprintFetchOptions, type FingerprintProfile, type FingerprintRequestBackend, } from "./types.ts"; export class SafeFingerprintAdapter implements FingerprintFetchAdapter { private readonly backends = new Map>(); private readonly policyClient: HttpClient; private readonly politeness: PolitenessController; private readonly robots: RobotsCache; constructor( private readonly factory: FingerprintBackendFactory, private readonly profile: FingerprintProfile, private readonly clientOptions: HttpClientOptions, ) { this.policyClient = new HttpClient(clientOptions); this.politeness = new PolitenessController({ globalConcurrency: clientOptions.globalConcurrency, perHostConcurrency: clientOptions.perHostConcurrency, }); this.robots = new RobotsCache({ userAgent: clientOptions.userAgent ?? DEFAULT_USER_AGENT, fetchText: (url, signal) => loadRobotsText(this.policyClient, url, signal), }); } async fetch( url: string | URL, options: FingerprintFetchOptions = {}, signal?: AbortSignal, ): Promise { assertSupportedFingerprintOptions({ ...this.profile, ...options }); if (this.clientOptions.fingerprintTrustLevel === "untrusted") { throw new UrlSafetyError( "FINGERPRINT_UNTRUSTED_URL", "mode: fingerprint cannot fully prevent DNS rebinding for untrusted URLs. Use mode: 'browser' or set fingerprintTrustLevel: 'trusted'.", url.toString(), ); } const initialSafe = await assertSafeFetchUrl(url, this.clientOptions); try { return await followRedirects({ initialSafe, maxRedirects: options.maxRedirects ?? this.clientOptions.maxRedirects ?? 5, fetchRequest: (safe) => fetchWithRequestPolicy({ safe, respectRobots: options.respectRobots, robots: this.robots, politeness: this.politeness, userAgent: this.clientOptions.userAgent ?? DEFAULT_USER_AGENT, signal, fetch: () => this.fetchOnce(safe, options, signal), }), resolveSafeUrl: (nextUrl) => assertSafeFetchUrl(nextUrl, this.clientOptions), }); } catch (error) { throw fingerprintFetchError(error, initialSafe.normalizedUrl, options); } } private async fetchOnce( safe: SafeUrlResult, options: FingerprintFetchOptions, parentSignal: AbortSignal | undefined, ): Promise { const timeoutMs = (options.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS) * 1_000; const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; const { signal, cleanup } = withTimeout(parentSignal, timeoutMs); try { const requestedProxy = options.proxy ?? resolveEnvProxyForUrl(safe.normalizedUrl) ?? this.profile.proxy; const effectiveProxy = validateFingerprintProxy(requestedProxy, safe.url); const backend = await this.backendFor(safe.url.host, effectiveProxy); const secondSafe = await revalidateDns(safe, this.clientOptions); // Load session and merge cookies into outgoing headers const session = options.sessionId ? await getOrCreateSession(options.sessionId, this.clientOptions.storage) : undefined; const baseHeaders = browserHeaders( this.clientOptions.userAgent ?? DEFAULT_USER_AGENT, options.headers, ); const mergedHeaders = session ? mergeSessionHeaders( session, safe.url.hostname, safe.url.pathname, safe.url.protocol === "https:" ? "https" : "http", baseHeaders, ) : baseHeaders; const response = await backend.fetchOnce( safe.normalizedUrl, { method: options.method === "HEAD" ? "HEAD" : "GET", headers: mergedHeaders, timeoutMs, maxBytes, browserProfile: options.browserProfile ?? this.profile.browserProfile ?? "chrome", osProfile: options.osProfile ?? this.profile.osProfile ?? "default", }, signal, ); const result = await materializeBackendResponse( safe.normalizedUrl, response, options, maxBytes, ); // Persist Set-Cookie back to session if (session && response.headers) { setCookiesFromResponse(session, response.headers, safe.url); } result.diagnostics = { fingerprintRebindingMitigation: { strategy: "double-resolve", preflightAddresses: safe.checkedAddresses, connectAddresses: secondSafe?.checkedAddresses ?? [], }, }; return result; } finally { cleanup(); } } private backendFor(host: string, proxy?: string): Promise { const effectiveProxy = proxy ?? this.profile.proxy; const key = JSON.stringify({ browserProfile: this.profile.browserProfile ?? "chrome", osProfile: this.profile.osProfile ?? "default", proxy: effectiveProxy, host, }); const existing = this.backends.get(key); if (existing) return existing; const backend = Promise.resolve( this.factory({ browserProfile: this.profile.browserProfile ?? "chrome", osProfile: this.profile.osProfile ?? "default", proxy: effectiveProxy, host, }), ); this.backends.set(key, backend); return backend; } } export async function materializeBackendResponse( url: string, response: FingerprintBackendResponse, options: FingerprintFetchOptions, maxBytes: number, ): Promise { const headers = normalizeHeaders(response.headers ?? {}); if (response.body && typeof response.body === "object" && "getReader" in response.body) { return await materializeFetchStreamResponse({ url, status: response.status, statusText: response.statusText, headers, body: response.body as unknown as AsyncIterable, maxBytes, options, discardBody: async () => { await (response.body as ReadableStream).cancel(); }, }); } const body = Buffer.isBuffer(response.body) ? response.body : Buffer.from(response.body ?? ""); return await materializeFetchBufferResponse({ url, status: response.status, statusText: response.statusText, headers, body, maxBytes, options, }); } async function revalidateDns( safe: SafeUrlResult, options: HttpClientOptions, ): Promise { if (safe.checkedAddresses.length === 0) return undefined; const second = await assertSafeFetchUrl(safe.normalizedUrl, options); if (second.checkedAddresses.length === 0) return second; const firstSet = new Set(safe.checkedAddresses); const secondSet = new Set(second.checkedAddresses); if (firstSet.size !== secondSet.size || ![...firstSet].every((ip) => secondSet.has(ip))) { throw new UrlSafetyError( "DNS_REBINDING_DETECTED", `DNS resolved to different IPs between preflight (${[...firstSet].join(", ")}) and connect-time (${[...secondSet].join(", ")}) check. Potential DNS rebinding attack.`, safe.normalizedUrl, ); } return second; } function fingerprintFetchError(error: unknown, url: string, options: FingerprintFetchOptions) { return httpClientErrorFromUnknown(error, url, options, { code: "FINGERPRINT_FETCH_FAILED", phase: "fingerprint", message: "Fingerprint fetch failed", }); } function validateFingerprintProxy(proxy: string | undefined, targetUrl: URL): string | undefined { if (!proxy) { return undefined; } const proxyUrl = validateProxyUrl(proxy); if (!isSocksProxyUrl(proxyUrl)) { return proxy; } const targetHost = stripIpv6Brackets(targetUrl.hostname); const targetFamily = isIP(targetHost); if (targetFamily === 0) { throw new HttpClientError({ code: "UNSUPPORTED_PROXY_SCHEME", phase: "proxy", message: "SOCKS proxies in fingerprint mode require proxy-side DNS for hostname targets; use mode: 'fast'/'readable' or an HTTP(S) proxy.", retryable: false, url: targetUrl.toString(), }); } if (proxyUrl.protocol === "socks4:" && targetFamily !== 4) { throw new HttpClientError({ code: "UNSUPPORTED_PROXY_SCHEME", phase: "proxy", message: "SOCKS4 proxies in fingerprint mode only support IPv4 target URLs.", retryable: false, url: targetUrl.toString(), }); } return proxy; } function stripIpv6Brackets(hostname: string): string { return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; } function browserHeaders( userAgent: string, headers: Record = {}, ): Record { return { "user-agent": userAgent, accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "accept-language": "en-US,en;q=0.9", "upgrade-insecure-requests": "1", ...headers, }; } /** @internal exported for testing only */ export function setCookiesFromResponse( session: FetchSession, headers: Record, url: URL, ): void { const raw = headers["set-cookie"]; if (!raw) return; const cookies = Array.isArray(raw) ? raw : [raw]; updateSessionCookies(session, cookies, url.hostname, url.pathname); }