import { completeRateLimitedResponse } from './async'; /** * Read a response body while enforcing a limit on the bytes received over the * wire. `Response.text()` cannot do this safely because it buffers the entire * response and a string's UTF-16 length is not its byte length. */ export async function readResponseTextWithinLimit( response: Response, options: { maxBytes: number; label: string }, ): Promise { const { maxBytes, label } = options; const advertisedLength = response.headers.get('content-length'); // Content-Length is only an early-rejection optimization. The stream is // always counted below because headers may be absent, malformed, or false. if (advertisedLength && /^\d+$/.test(advertisedLength) && BigInt(advertisedLength) > BigInt(maxBytes)) { try { await response.body?.cancel(); } catch { // The size error is more useful than a connection teardown failure. } finally { completeRateLimitedResponse(response); } throw new Error(`${label} too large: ${advertisedLength} bytes (max: ${maxBytes})`); } if (!response.body) { completeRateLimitedResponse(response); return ''; } const reader = response.body.getReader(); let bytesRead = 0; const chunks: Uint8Array[] = []; try { while (true) { const { done, value } = await reader.read(); if (done) break; bytesRead += value.byteLength; if (bytesRead > maxBytes) { // Stop pulling data immediately. This also closes the underlying // response body when the request's controller is no longer available. try { await reader.cancel(); } catch { // Preserve the limit violation when the underlying cancellation fails. } throw new Error(`${label} too large: ${bytesRead} bytes (max: ${maxBytes})`); } chunks.push(value); } const bytes = new Uint8Array(bytesRead); let offset = 0; for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } return decodeResponseBytes(response, bytes); } finally { reader.releaseLock(); completeRateLimitedResponse(response); } } function textEncodingForResponse(response: Response, bytes: Uint8Array): string { const bomEncoding = bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf ? 'utf-8' : bytes[0] === 0xff && bytes[1] === 0xfe ? 'utf-16le' : bytes[0] === 0xfe && bytes[1] === 0xff ? 'utf-16be' : bytes[0] === 0x3c && bytes[1] === 0x00 && bytes[2] === 0x3f && bytes[3] === 0x00 ? 'utf-16le' : bytes[0] === 0x00 && bytes[1] === 0x3c && bytes[2] === 0x00 && bytes[3] === 0x3f ? 'utf-16be' : undefined; if (bomEncoding) return bomEncoding; const contentTypeEncoding = response.headers.get('content-type')?.match(/\bcharset\s*=\s*["']?([^\s;"']+)/i)?.[1]; if (contentTypeEncoding) return contentTypeEncoding; // XML declarations are ASCII-compatible at the start of XML documents. A // Latin-1 view lets us find the declaration without assuming UTF-8 payload // text (and without decoding beyond the byte cap enforced below). const prefix = String.fromCharCode(...bytes.subarray(0, Math.min(bytes.length, 1024))); return prefix.match(/<\?xml\s+[^>]*\bencoding\s*=\s*["']([^"']+)["']/i)?.[1] ?? 'utf-8'; } function decodeResponseBytes(response: Response, bytes: Uint8Array): string { const encoding = textEncodingForResponse(response, bytes); try { return new TextDecoder(encoding).decode(bytes); } catch { // An invalid server charset should not make a bounded response unreadable. return new TextDecoder().decode(bytes); } }