import dns from 'node:dns'; import http from 'node:http'; import https from 'node:https'; import type { LookupFunction } from 'node:net'; import { assertPublicHttpUrl, isBlockedIpAddress, isRedirectStatus, normalizeUrlHostname, resolveRedirectUrl, UnsafeOutboundUrlError, } from '@shared_libs/security/outbound-url-policy'; type NodeSafeFetchOptions = { maxRedirects?: number; maxResponseBytes?: number; /** Return after validated headers so the caller can enforce a body deadline. */ streamResponseBody?: boolean; truncateResponseBody?: boolean; sensitiveHeaders?: Iterable; validateUrl?: (url: URL) => void; }; type PreparedBody = { body: Buffer | string | undefined; headers: Headers; }; export { UnsafeOutboundUrlError }; const NULL_BODY_STATUS_CODES = new Set([204, 205, 304]); const TRANSPORT_OWNED_REQUEST_HEADERS = new Set([ 'connection', 'content-length', 'host', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailer', 'transfer-encoding', 'upgrade', 'forwarded', 'x-forwarded-for', 'x-forwarded-host', 'x-forwarded-proto', ]); const CROSS_ORIGIN_REDIRECT_SAFE_HEADERS = new Set([ 'accept', 'accept-language', 'content-language', 'content-type', ]); function cloneHeaders(headers: RequestInit['headers']): Headers { return new Headers(headers); } function deleteHeader(headers: Headers, name: string) { if (headers.has(name)) headers.delete(name); } function removeTransportOwnedHeaders(headers: Headers): void { for (const header of TRANSPORT_OWNED_REQUEST_HEADERS) { deleteHeader(headers, header); } } function removeUnsafeCrossOriginRedirectHeaders(headers: Headers): void { const namesToDelete: string[] = []; for (const [name] of headers) { if (!CROSS_ORIGIN_REDIRECT_SAFE_HEADERS.has(name.toLowerCase())) { namesToDelete.push(name); } } for (const name of namesToDelete) { headers.delete(name); } } function withContentLength( body: Buffer | string | undefined, headers: Headers, ): PreparedBody { removeTransportOwnedHeaders(headers); if (body !== undefined) { headers.set( 'content-length', String(typeof body === 'string' ? Buffer.byteLength(body) : body.length), ); } return { body, headers }; } async function prepareBody(init: RequestInit): Promise { const headers = cloneHeaders(init.headers); const body = init.body; if (body === undefined || body === null) { return withContentLength(undefined, headers); } if (typeof body === 'string') { return withContentLength(body, headers); } if (body instanceof URLSearchParams) { if (!headers.has('content-type')) { headers.set( 'content-type', 'application/x-www-form-urlencoded;charset=UTF-8', ); } return withContentLength(body.toString(), headers); } if (body instanceof ArrayBuffer) { return withContentLength(Buffer.from(body), headers); } if (ArrayBuffer.isView(body)) { return withContentLength( Buffer.from(body.buffer, body.byteOffset, body.byteLength), headers, ); } if (typeof Blob !== 'undefined' && body instanceof Blob) { return withContentLength(Buffer.from(await body.arrayBuffer()), headers); } throw new TypeError( 'safeOutboundFetch does not support streaming or multipart request bodies.', ); } function headersToRecord(headers: Headers): Record { return Object.fromEntries(headers.entries()); } function validateResolvedAddress(address: string) { if (isBlockedIpAddress(address)) { throw new UnsafeOutboundUrlError( `Resolved address "${address}" is not allowed.`, ); } } function responseMustNotHaveBody(method: string, status: number): boolean { return method === 'HEAD' || NULL_BODY_STATUS_CODES.has(status); } const safeLookup: LookupFunction = (hostname, options, callback) => { const lookupOptions: dns.LookupAllOptions = { all: true, verbatim: true, ...(typeof options.family === 'number' ? { family: options.family } : {}), ...(typeof options.hints === 'number' ? { hints: options.hints } : {}), }; dns.lookup(hostname, lookupOptions, (error, records) => { if (error) { callback(error, '', 0); return; } if (!records.length) { callback( Object.assign(new Error(`Could not resolve host "${hostname}".`), { code: 'ENOTFOUND', }) as NodeJS.ErrnoException, '', 0, ); return; } try { for (const record of records) { validateResolvedAddress(record.address); } } catch (validationError) { callback(validationError as NodeJS.ErrnoException, '', 0); return; } if (options.all) { callback(null, records); return; } callback(null, records[0]!.address, records[0]!.family); }); }; function createRequest( url: URL, init: RequestInit, prepared: PreparedBody, maxResponseBytes: number | undefined, truncateResponseBody: boolean, ): Promise { return new Promise((resolve, reject) => { const transport = url.protocol === 'https:' ? https : http; const method = String(init.method ?? 'GET').toUpperCase(); const request = transport.request( url, { method, headers: headersToRecord(prepared.headers), agent: false, lookup: safeLookup, signal: init.signal ?? undefined, }, (response) => { const status = response.statusCode ?? 0; const noBodyResponse = responseMustNotHaveBody(method, status); const contentLength = Number(response.headers['content-length']); if ( !noBodyResponse && maxResponseBytes !== undefined && Number.isFinite(contentLength) && contentLength > maxResponseBytes && !truncateResponseBody ) { response.resume(); reject( new Error(`Response body exceeds ${maxResponseBytes} byte limit.`), ); return; } if (noBodyResponse) { response.resume(); resolve( new Response(null, { status, statusText: response.statusMessage, headers: response.headers as HeadersInit, }), ); return; } let receivedBytes = 0; let bodySettled = false; const body = new ReadableStream({ start(controller) { response.on('data', (chunk) => { if (bodySettled) return; const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); const previousReceivedBytes = receivedBytes; receivedBytes += buffer.byteLength; if ( maxResponseBytes !== undefined && receivedBytes > maxResponseBytes && !truncateResponseBody ) { bodySettled = true; const error = new Error( `Response body exceeds ${maxResponseBytes} byte limit.`, ); controller.error(error); response.destroy(error); return; } if (maxResponseBytes !== undefined && truncateResponseBody) { const remainingBytes = Math.max( 0, maxResponseBytes - previousReceivedBytes, ); if (remainingBytes > 0) { controller.enqueue(buffer.subarray(0, remainingBytes)); } if (receivedBytes >= maxResponseBytes) { bodySettled = true; controller.close(); response.destroy(); } return; } controller.enqueue(buffer); }); response.on('error', (error) => { if (bodySettled) return; bodySettled = true; controller.error(error); }); response.on('end', () => { if (bodySettled) return; bodySettled = true; controller.close(); }); }, cancel(reason) { if (bodySettled) return; bodySettled = true; response.destroy(reason instanceof Error ? reason : undefined); }, }); resolve( new Response(body, { status, statusText: response.statusMessage, headers: response.headers as HeadersInit, }), ); }, ); request.on('error', reject); if (prepared.body !== undefined) { request.write(prepared.body); } request.end(); }); } function initForRedirect( init: RequestInit, from: URL, to: URL, status: number, ): RequestInit { const headers = cloneHeaders(init.headers); let method = String(init.method ?? 'GET').toUpperCase(); let body = init.body; if ( status === 303 || ((status === 301 || status === 302) && method === 'POST') ) { method = 'GET'; body = undefined; deleteHeader(headers, 'content-length'); deleteHeader(headers, 'content-type'); } if (from.origin !== to.origin) { removeUnsafeCrossOriginRedirectHeaders(headers); } return { ...init, method, body, headers, redirect: 'manual', }; } export async function safeOutboundFetch( input: string | URL, init: RequestInit = {}, options: NodeSafeFetchOptions = {}, ): Promise { const redirectMode = init.redirect ?? 'follow'; const maxRedirects = options.maxRedirects ?? 10; const maxResponseBytes = options.maxResponseBytes; const truncateResponseBody = options.truncateResponseBody === true; const validateUrl = options.validateUrl; let currentUrl = assertPublicHttpUrl(input); let currentInit: RequestInit = { ...init, redirect: 'manual' }; for ( let redirectCount = 0; redirectCount <= maxRedirects; redirectCount += 1 ) { const hostname = normalizeUrlHostname(currentUrl.hostname); validateUrl?.(currentUrl); if (isBlockedIpAddress(hostname)) { throw new UnsafeOutboundUrlError( `Target host "${hostname}" is not allowed.`, ); } const response = await createRequest( currentUrl, currentInit, await prepareBody(currentInit), maxResponseBytes, truncateResponseBody, ); Object.defineProperty(response, 'url', { configurable: true, value: currentUrl.toString(), }); if (!isRedirectStatus(response.status)) { if (options.streamResponseBody) return response; const body = response.body ? await response.arrayBuffer() : null; const buffered = new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers, }); Object.defineProperty(buffered, 'url', { configurable: true, value: response.url, }); return buffered; } if (redirectMode === 'error') { void response.body?.cancel().catch(() => undefined); throw new Error( `Redirect blocked while fetching ${currentUrl.toString()}.`, ); } if (redirectMode !== 'follow') { return response; } const location = response.headers.get('location'); if (!location) { return response; } if (redirectCount === maxRedirects) { void response.body?.cancel().catch(() => undefined); throw new Error( `Too many redirects while fetching ${currentUrl.toString()}.`, ); } void response.body?.cancel().catch(() => undefined); const nextUrl = resolveRedirectUrl(location, currentUrl); currentInit = initForRedirect( currentInit, currentUrl, nextUrl, response.status, ); currentUrl = nextUrl; } throw new Error( `Too many redirects while fetching ${currentUrl.toString()}.`, ); }