{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../../src/core/web-research/fetch.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,eAAe,EAA6B,MAAM,cAAc,CAAC;AAC/E,OAAO,EAAE,KAAK,2BAA2B,EAA4B,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAChH,OAAO,EAGN,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EAEtB,KAAK,oBAAoB,EACzB,MAAM,YAAY,CAAC;AAGpB,UAAU,cAAc;IACvB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,2BAA2B,CAAC,EAAE,2BAA2B,CAAC;IAC1D,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACjB;AAwSD,qBAAa,gBAAgB;IAK3B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IANzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAa;IAEjC,YACkB,MAAM,EAAE,iBAAiB,EACzB,SAAS,EAAE,oBAAoB,EAC/B,OAAO,GAAE,cAAmB,EAK7C;IAED,IAAI,gBAAgB,IAAI,OAAO,CAE9B;YAEa,QAAQ;IAoDhB,KAAK,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA4F/D;CACD","sourcesContent":["import { createHash } from \"node:crypto\";\nimport { request as requestHttp } from \"node:http\";\nimport { request as requestHttps } from \"node:https\";\nimport type { LookupFunction } from \"node:net\";\nimport { brotliDecompressSync, gunzipSync, inflateSync } from \"node:zlib\";\nimport { Readability } from \"@mozilla/readability\";\nimport { parseHTML } from \"linkedom\";\nimport TurndownService from \"turndown\";\nimport { extractText, getDocumentProxy } from \"unpdf\";\nimport { type BrowserRenderer, PlaywrightBrowserRenderer } from \"./browser.js\";\nimport { type AdministrativeNetworkPolicy, resolveAndValidateWebUrl, type WebDnsResolver } from \"./security.js\";\nimport {\n\ttype EvidencePassage,\n\ttype WebEvidenceRecord,\n\ttype WebFetchRequest,\n\ttype WebFetchResponse,\n\ttype WebResearchConfig,\n\tWebResearchError,\n\ttype WebResearchTelemetry,\n} from \"./types.js\";\nimport { canonicalizeWebUrl, sanitizeUrlForDiagnostics } from \"./url.js\";\n\ninterface FetcherOptions {\n\tresolver?: WebDnsResolver;\n\tadministrativeNetworkPolicy?: AdministrativeNetworkPolicy;\n\tbrowserRenderer?: BrowserRenderer;\n\tnow?: () => Date;\n}\n\ninterface RawResponse {\n\trequestedUrl: string;\n\tfinalUrl: string;\n\tstatus: number;\n\theaders: Record<string, string | string[] | undefined>;\n\tbody: Buffer;\n\tbytesDownloaded: number;\n}\n\ninterface ExtractedContent {\n\tcontent: string;\n\ttitle?: string;\n\tauthor?: string;\n\tpublishedAt?: string;\n\tcanonicalUrl?: string;\n\toutboundLinks: string[];\n\textractor: WebEvidenceRecord[\"extractor\"];\n\tpageCount?: number;\n}\n\ninterface PdfMetadata {\n\tinfo?: { Title?: unknown; Author?: unknown; CreationDate?: unknown };\n}\n\nfunction sha256(value: string | Uint8Array): string {\n\treturn createHash(\"sha256\").update(value).digest(\"hex\");\n}\n\nfunction normalizeContent(value: string): string {\n\treturn value\n\t\t.replace(/\\r\\n?/g, \"\\n\")\n\t\t.replace(/[ \\t]+\\n/g, \"\\n\")\n\t\t.replace(/\\n{3,}/g, \"\\n\\n\")\n\t\t.trim();\n}\n\nfunction headerValue(value: string | string[] | undefined): string {\n\treturn Array.isArray(value) ? value.join(\", \") : (value ?? \"\");\n}\n\nfunction mimeType(headers: RawResponse[\"headers\"]): string {\n\treturn headerValue(headers[\"content-type\"]).split(\";\", 1)[0].trim().toLowerCase();\n}\n\nfunction decodeText(body: Buffer, contentType: string): string {\n\tconst charset = /charset\\s*=\\s*[\"']?([^;\"'\\s]+)/i.exec(contentType)?.[1]?.toLowerCase() ?? \"utf-8\";\n\ttry {\n\t\treturn new TextDecoder(charset).decode(body);\n\t} catch {\n\t\treturn new TextDecoder(\"utf-8\").decode(body);\n\t}\n}\n\nfunction decompressBody(body: Buffer, encoding: string, maximum: number): Buffer {\n\tconst normalized = encoding.trim().toLowerCase();\n\ttry {\n\t\tif (!normalized || normalized === \"identity\") return body;\n\t\tif (normalized === \"gzip\" || normalized === \"x-gzip\") return gunzipSync(body, { maxOutputLength: maximum });\n\t\tif (normalized === \"deflate\") return inflateSync(body, { maxOutputLength: maximum });\n\t\tif (normalized === \"br\") return brotliDecompressSync(body, { maxOutputLength: maximum });\n\t\tthrow new WebResearchError(\"CONTENT_TYPE_UNSUPPORTED\", `Unsupported content encoding: ${normalized}`);\n\t} catch (error) {\n\t\tif (error instanceof WebResearchError) throw error;\n\t\tthrow new WebResearchError(\n\t\t\t\"RESPONSE_TOO_LARGE\",\n\t\t\t\"Compressed response exceeded the decompression limit or was invalid\",\n\t\t\t{\n\t\t\t\tcause: error,\n\t\t\t},\n\t\t);\n\t}\n}\n\nfunction createPinnedLookup(addresses: Array<{ address: string; family: number }>): LookupFunction {\n\treturn (_hostname, options, callback) => {\n\t\tif (options.all) {\n\t\t\tcallback(null, addresses);\n\t\t\treturn;\n\t\t}\n\t\tconst address = addresses[0];\n\t\tcallback(null, address.address, address.family);\n\t};\n}\n\nasync function readHttpResponse(\n\turl: URL,\n\taddresses: Array<{ address: string; family: number }>,\n\tconfig: WebResearchConfig,\n\tsignal: AbortSignal,\n): Promise<{ status: number; headers: RawResponse[\"headers\"]; body: Buffer }> {\n\treturn new Promise((resolve, reject) => {\n\t\tconst request = (url.protocol === \"https:\" ? requestHttps : requestHttp)(\n\t\t\turl,\n\t\t\t{\n\t\t\t\tmethod: \"GET\",\n\t\t\t\theaders: {\n\t\t\t\t\tAccept:\n\t\t\t\t\t\t\"text/html,text/plain,text/markdown,application/json,application/xml,text/xml,application/pdf;q=0.9,*/*;q=0.1\",\n\t\t\t\t\t\"Accept-Encoding\": \"gzip, deflate, br\",\n\t\t\t\t\t\"User-Agent\": config.userAgent,\n\t\t\t\t},\n\t\t\t\tlookup: createPinnedLookup(addresses),\n\t\t\t\tsignal,\n\t\t\t},\n\t\t\t(response) => {\n\t\t\t\tconst status = response.statusCode ?? 0;\n\t\t\t\tconst length = Number(headerValue(response.headers[\"content-length\"]));\n\t\t\t\tif (Number.isFinite(length) && length > config.maxResponseBytes) {\n\t\t\t\t\tresponse.destroy();\n\t\t\t\t\treject(\n\t\t\t\t\t\tnew WebResearchError(\"RESPONSE_TOO_LARGE\", \"Response Content-Length exceeds the configured limit\"),\n\t\t\t\t\t);\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\tconst chunks: Buffer[] = [];\n\t\t\t\tlet bytes = 0;\n\t\t\t\tresponse.on(\"data\", (chunk: Buffer) => {\n\t\t\t\t\tbytes += chunk.length;\n\t\t\t\t\tif (bytes > config.maxResponseBytes) {\n\t\t\t\t\t\tresponse.destroy(\n\t\t\t\t\t\t\tnew WebResearchError(\"RESPONSE_TOO_LARGE\", \"Response exceeded the configured size limit\"),\n\t\t\t\t\t\t);\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tchunks.push(chunk);\n\t\t\t\t});\n\t\t\t\tresponse.on(\"end\", () => resolve({ status, headers: response.headers, body: Buffer.concat(chunks) }));\n\t\t\t\tresponse.on(\"error\", reject);\n\t\t\t},\n\t\t);\n\t\trequest.on(\"error\", reject);\n\t\trequest.end();\n\t});\n}\n\nfunction isRedirect(status: number): boolean {\n\treturn status === 301 || status === 302 || status === 303 || status === 307 || status === 308;\n}\n\nfunction ensureSupportedMime(type: string, body: Buffer): string {\n\tif (type === \"application/octet-stream\" && body.subarray(0, 5).toString(\"ascii\") === \"%PDF-\")\n\t\treturn \"application/pdf\";\n\tif (!type && /^\\s*</.test(body.subarray(0, 512).toString(\"utf8\"))) return \"text/html\";\n\tconst supported = new Set([\n\t\t\"text/html\",\n\t\t\"application/xhtml+xml\",\n\t\t\"text/plain\",\n\t\t\"text/markdown\",\n\t\t\"text/x-markdown\",\n\t\t\"application/json\",\n\t\t\"application/xml\",\n\t\t\"text/xml\",\n\t\t\"application/pdf\",\n\t]);\n\tif (!supported.has(type))\n\t\tthrow new WebResearchError(\"CONTENT_TYPE_UNSUPPORTED\", `Unsupported response content type: ${type || \"missing\"}`);\n\treturn type;\n}\n\ntype ParsedDocument = ReturnType<typeof parseHTML>[\"document\"];\n\nfunction canonicalLink(document: ParsedDocument, baseUrl: string): string | undefined {\n\tconst href = document.querySelector('link[rel=\"canonical\"]')?.getAttribute(\"href\");\n\tif (!href) return undefined;\n\ttry {\n\t\treturn canonicalizeWebUrl(new URL(href, baseUrl).toString());\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nfunction outboundLinks(document: ParsedDocument, baseUrl: string): string[] {\n\tconst links = new Set<string>();\n\tfor (const element of document.querySelectorAll(\"a[href]\")) {\n\t\tconst href = element.getAttribute(\"href\");\n\t\tif (!href) continue;\n\t\ttry {\n\t\t\tconst url = new URL(href, baseUrl);\n\t\t\tif (url.protocol === \"http:\" || url.protocol === \"https:\") links.add(canonicalizeWebUrl(url.toString()));\n\t\t} catch {\n\t\t\t// Ignore malformed outbound links from untrusted pages.\n\t\t}\n\t}\n\treturn [...links].sort();\n}\n\nfunction extractHtml(\n\thtml: string,\n\tbaseUrl: string,\n\textractor: \"readability\" | \"playwright\" = \"readability\",\n): ExtractedContent {\n\tconst parseable = /<html[\\s>]/i.test(html) ? html : `<!doctype html><html><body>${html}</body></html>`;\n\tconst { document } = parseHTML(parseable);\n\tfor (const selector of [\"script\", \"style\", \"noscript\", \"template\", \"nav\", \"aside\", \"form\", \"dialog\"]) {\n\t\tfor (const element of document.querySelectorAll(selector)) element.remove();\n\t}\n\tconst canonicalUrl = canonicalLink(document, baseUrl);\n\tconst links = outboundLinks(document, baseUrl);\n\tconst readabilityDocument = document.cloneNode(true) as unknown as ConstructorParameters<typeof Readability>[0];\n\tconst readable = new Readability(readabilityDocument, {\n\t\tcharThreshold: 120,\n\t\tmaxElemsToParse: 50_000,\n\t}).parse();\n\tconst selectedHtml = readable?.content || document.body?.innerHTML || \"\";\n\tconst turndown = new TurndownService({ bulletListMarker: \"-\", codeBlockStyle: \"fenced\", emDelimiter: \"_\" });\n\tturndown.remove([\"script\", \"style\", \"noscript\", \"template\", \"iframe\"]);\n\tconst content = normalizeContent(turndown.turndown(selectedHtml));\n\treturn {\n\t\tcontent,\n\t\ttitle: readable?.title?.trim() || document.title?.trim() || undefined,\n\t\tauthor: readable?.byline?.trim() || undefined,\n\t\tpublishedAt: readable?.publishedTime?.trim() || undefined,\n\t\tcanonicalUrl,\n\t\toutboundLinks: links,\n\t\textractor,\n\t};\n}\n\nfunction canonicalJson(value: unknown): unknown {\n\tif (Array.isArray(value)) return value.map(canonicalJson);\n\tif (value !== null && typeof value === \"object\") {\n\t\tconst output: Record<string, unknown> = {};\n\t\tfor (const key of Object.keys(value).sort()) output[key] = canonicalJson((value as Record<string, unknown>)[key]);\n\t\treturn output;\n\t}\n\treturn value;\n}\n\nasync function extractPdf(body: Buffer): Promise<ExtractedContent> {\n\tif (body.subarray(0, 5).toString(\"ascii\") !== \"%PDF-\") {\n\t\tthrow new WebResearchError(\"CONTENT_INVALID\", \"Response declared PDF content but lacks a PDF signature\");\n\t}\n\ttry {\n\t\tconst document = await getDocumentProxy(new Uint8Array(body));\n\t\tconst [textResult, metadata] = await Promise.all([\n\t\t\textractText(document, { mergePages: false }),\n\t\t\tdocument.getMetadata().catch(() => undefined) as Promise<PdfMetadata | undefined>,\n\t\t]);\n\t\tconst pages = textResult.text.map((page, index) => `<!-- page:${index + 1} -->\\n${normalizeContent(page)}`);\n\t\tconst content = normalizeContent(pages.join(\"\\n\\n\"));\n\t\tif (!content)\n\t\t\tthrow new WebResearchError(\"PDF_IMAGE_ONLY\", \"PDF contains no extractable text; OCR is not enabled\");\n\t\treturn {\n\t\t\tcontent,\n\t\t\ttitle: typeof metadata?.info?.Title === \"string\" ? metadata.info.Title : undefined,\n\t\t\tauthor: typeof metadata?.info?.Author === \"string\" ? metadata.info.Author : undefined,\n\t\t\tpublishedAt: typeof metadata?.info?.CreationDate === \"string\" ? metadata.info.CreationDate : undefined,\n\t\t\toutboundLinks: [],\n\t\t\textractor: \"pdf\",\n\t\t\tpageCount: textResult.totalPages,\n\t\t};\n\t} catch (error) {\n\t\tif (error instanceof WebResearchError) throw error;\n\t\tconst message = error instanceof Error ? error.message : String(error);\n\t\tif (/password|encrypted/i.test(message))\n\t\t\tthrow new WebResearchError(\"PDF_ENCRYPTED\", \"Encrypted PDF cannot be extracted\", { cause: error });\n\t\tthrow new WebResearchError(\"CONTENT_INVALID\", \"PDF is malformed or unsupported\", { cause: error });\n\t}\n}\n\nfunction buildPassages(content: string, query: string | undefined, isPdf: boolean): EvidencePassage[] {\n\tconst lines = content.split(\"\\n\");\n\tconst terms = (query?.toLowerCase().match(/[a-z0-9]{3,}/g) ?? []).filter(\n\t\t(term, index, values) => values.indexOf(term) === index,\n\t);\n\tconst candidates: Array<{ passage: EvidencePassage; score: number }> = [];\n\tfor (let start = 0; start < lines.length; start += 8) {\n\t\tconst end = Math.min(lines.length, start + 12);\n\t\tconst text = lines.slice(start, end).join(\"\\n\").trim();\n\t\tif (!text) continue;\n\t\tconst lower = text.toLowerCase();\n\t\tconst score = terms.reduce((sum, term) => sum + (lower.includes(term) ? 1 : 0), 0);\n\t\tconst pageMatch = isPdf ? /<!-- page:(\\d+) -->/.exec(text) : undefined;\n\t\tcandidates.push({\n\t\t\tpassage: {\n\t\t\t\tid: `passage-${start + 1}-${end}`,\n\t\t\t\ttext: text.slice(0, 3000),\n\t\t\t\tstartLine: start + 1,\n\t\t\t\tendLine: end,\n\t\t\t\tpage: pageMatch ? Number(pageMatch[1]) : undefined,\n\t\t\t},\n\t\t\tscore,\n\t\t});\n\t}\n\treturn candidates\n\t\t.sort((left, right) => right.score - left.score || (left.passage.startLine ?? 0) - (right.passage.startLine ?? 0))\n\t\t.slice(0, 4)\n\t\t.map(({ passage }) => passage);\n}\n\nfunction shouldRender(extracted: ExtractedContent): boolean {\n\treturn extracted.content.replace(/[#*_`>\\-\\s]/g, \"\").length < 240;\n}\n\nexport class SecureWebFetcher {\n\tprivate readonly browser: BrowserRenderer;\n\tprivate readonly now: () => Date;\n\n\tconstructor(\n\t\tprivate readonly config: WebResearchConfig,\n\t\tprivate readonly telemetry: WebResearchTelemetry,\n\t\tprivate readonly options: FetcherOptions = {},\n\t) {\n\t\tthis.browser =\n\t\t\toptions.browserRenderer ?? new PlaywrightBrowserRenderer(config.browserExecutablePath, config.fetchTimeoutMs);\n\t\tthis.now = options.now ?? (() => new Date());\n\t}\n\n\tget browserAvailable(): boolean {\n\t\treturn this.browser.available;\n\t}\n\n\tprivate async rawFetch(requestedUrl: string, signal: AbortSignal): Promise<RawResponse> {\n\t\tlet current = requestedUrl;\n\t\tfor (let redirect = 0; redirect <= this.config.maxRedirects; redirect++) {\n\t\t\tlet resolved: Awaited<ReturnType<typeof resolveAndValidateWebUrl>>;\n\t\t\ttry {\n\t\t\t\tresolved = await resolveAndValidateWebUrl(current, {\n\t\t\t\t\tresolver: this.options.resolver,\n\t\t\t\t\tadministrativeNetworkPolicy: this.options.administrativeNetworkPolicy,\n\t\t\t\t\tsignal,\n\t\t\t\t});\n\t\t\t} catch (error) {\n\t\t\t\tif (error instanceof WebResearchError && (error.code === \"DNS_BLOCKED\" || error.code === \"URL_BLOCKED\"))\n\t\t\t\t\tthis.telemetry.ssrfBlocks++;\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t\tconst response = await readHttpResponse(resolved.url, resolved.addresses, this.config, signal);\n\t\t\tif (isRedirect(response.status)) {\n\t\t\t\tconst location = headerValue(response.headers.location);\n\t\t\t\tif (!location) throw new WebResearchError(\"CONTENT_INVALID\", \"Redirect response is missing Location\");\n\t\t\t\tif (redirect === this.config.maxRedirects)\n\t\t\t\t\tthrow new WebResearchError(\"REDIRECT_LIMIT\", \"Maximum redirect count exceeded\");\n\t\t\t\tcurrent = new URL(location, resolved.url).toString();\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (response.status < 200 || response.status >= 300) {\n\t\t\t\tthrow new WebResearchError(\"PROVIDER_UNAVAILABLE\", `Web fetch failed with HTTP ${response.status}`, {\n\t\t\t\t\tsanitizedUrl: sanitizeUrlForDiagnostics(current),\n\t\t\t\t});\n\t\t\t}\n\t\t\tconst decompressed = decompressBody(\n\t\t\t\tresponse.body,\n\t\t\t\theaderValue(response.headers[\"content-encoding\"]),\n\t\t\t\tthis.config.maxDecompressedBytes,\n\t\t\t);\n\t\t\tif (decompressed.length > this.config.maxDecompressedBytes) {\n\t\t\t\tthrow new WebResearchError(\n\t\t\t\t\t\"RESPONSE_TOO_LARGE\",\n\t\t\t\t\t\"Decompressed response exceeded the configured size limit\",\n\t\t\t\t);\n\t\t\t}\n\t\t\treturn {\n\t\t\t\trequestedUrl,\n\t\t\t\tfinalUrl: resolved.url.toString(),\n\t\t\t\tstatus: response.status,\n\t\t\t\theaders: response.headers,\n\t\t\t\tbody: decompressed,\n\t\t\t\tbytesDownloaded: response.body.length,\n\t\t\t};\n\t\t}\n\t\tthrow new WebResearchError(\"REDIRECT_LIMIT\", \"Maximum redirect count exceeded\");\n\t}\n\n\tasync fetch(request: WebFetchRequest): Promise<WebFetchResponse> {\n\t\tconst started = performance.now();\n\t\tthis.telemetry.fetches++;\n\t\tconst timeout = AbortSignal.timeout(this.config.fetchTimeoutMs);\n\t\tconst signal = request.signal ? AbortSignal.any([request.signal, timeout]) : timeout;\n\t\ttry {\n\t\t\tconst raw = await this.rawFetch(request.url, signal);\n\t\t\tconst type = ensureSupportedMime(mimeType(raw.headers), raw.body);\n\t\t\tconst contentType = headerValue(raw.headers[\"content-type\"]);\n\t\t\tlet extracted: ExtractedContent;\n\t\t\tlet sourceHtml: string | undefined;\n\t\t\tif (request.mode === \"pdf\" || type === \"application/pdf\") {\n\t\t\t\textracted = await extractPdf(raw.body);\n\t\t\t} else if (request.mode === \"json\" || type === \"application/json\") {\n\t\t\t\tconst parsed = JSON.parse(decodeText(raw.body, contentType)) as unknown;\n\t\t\t\textracted = {\n\t\t\t\t\tcontent: JSON.stringify(canonicalJson(parsed), null, 2),\n\t\t\t\t\toutboundLinks: [],\n\t\t\t\t\textractor: \"json\",\n\t\t\t\t};\n\t\t\t} else if (request.mode === \"xml\" || type === \"application/xml\" || type === \"text/xml\") {\n\t\t\t\textracted = {\n\t\t\t\t\tcontent: normalizeContent(decodeText(raw.body, contentType)),\n\t\t\t\t\toutboundLinks: [],\n\t\t\t\t\textractor: \"xml\",\n\t\t\t\t};\n\t\t\t} else if (request.mode === \"text\" || type === \"text/plain\" || type.includes(\"markdown\")) {\n\t\t\t\textracted = {\n\t\t\t\t\tcontent: normalizeContent(decodeText(raw.body, contentType)),\n\t\t\t\t\toutboundLinks: [],\n\t\t\t\t\textractor: \"text\",\n\t\t\t\t};\n\t\t\t} else {\n\t\t\t\tsourceHtml = decodeText(raw.body, contentType);\n\t\t\t\textracted = extractHtml(sourceHtml, raw.finalUrl);\n\t\t\t}\n\t\t\tlet rendered = false;\n\t\t\tif (\n\t\t\t\tsourceHtml !== undefined &&\n\t\t\t\t(request.render === \"always\" || (request.render !== \"never\" && shouldRender(extracted)))\n\t\t\t) {\n\t\t\t\tconst renderedHtml = await this.browser.renderHtml(sourceHtml, signal);\n\t\t\t\textracted = extractHtml(renderedHtml, raw.finalUrl, \"playwright\");\n\t\t\t\trendered = true;\n\t\t\t\tthis.telemetry.renderFallbacks++;\n\t\t\t}\n\t\t\tif (!extracted.content)\n\t\t\t\tthrow new WebResearchError(\"CONTENT_INVALID\", \"Fetched content contained no extractable text\");\n\t\t\tconst completeContent = normalizeContent(extracted.content);\n\t\t\tconst hash = sha256(completeContent);\n\t\t\tconst canonicalUrl = extracted.canonicalUrl ?? canonicalizeWebUrl(raw.finalUrl);\n\t\t\tconst evidenceId = `web-${sha256(`${canonicalUrl}\\n${hash}`).slice(0, 20)}`;\n\t\t\tconst maxCharacters = Math.max(500, Math.min(request.maxCharacters ?? 12_000, 50_000));\n\t\t\tconst truncated = completeContent.length > maxCharacters;\n\t\t\tconst content = completeContent.slice(0, maxCharacters);\n\t\t\tconst relevantPassages = buildPassages(completeContent, request.passageQuery, extracted.extractor === \"pdf\");\n\t\t\tconst evidence: WebEvidenceRecord = {\n\t\t\t\tevidenceId,\n\t\t\t\tsourceType: \"web\",\n\t\t\t\trequestedUrl: canonicalizeWebUrl(raw.requestedUrl),\n\t\t\t\tfinalUrl: canonicalizeWebUrl(raw.finalUrl),\n\t\t\t\tcanonicalUrl,\n\t\t\t\ttitle: extracted.title,\n\t\t\t\tauthor: extracted.author,\n\t\t\t\tretrievedAt: this.now().toISOString(),\n\t\t\t\tpublishedAt: extracted.publishedAt,\n\t\t\t\tcontentType: type,\n\t\t\t\textractor: extracted.extractor,\n\t\t\t\tcontentSha256: hash,\n\t\t\t\tcompleteContentLocation: `session:tool-result:${evidenceId}`,\n\t\t\t\tcompleteContent,\n\t\t\t\trelevantPassages,\n\t\t\t\toutboundLinks: extracted.outboundLinks,\n\t\t\t\tbytesDownloaded: raw.bytesDownloaded,\n\t\t\t\tbytesExtracted: Buffer.byteLength(completeContent),\n\t\t\t\ttruncated,\n\t\t\t\tpageCount: extracted.pageCount,\n\t\t\t\tuntrusted: true,\n\t\t\t};\n\t\t\tthis.telemetry.bytesDownloaded += evidence.bytesDownloaded;\n\t\t\tthis.telemetry.bytesExtracted += evidence.bytesExtracted;\n\t\t\tthis.telemetry.evidenceRecords++;\n\t\t\treturn { evidence, content, rendered, durationMs: Math.round(performance.now() - started) };\n\t\t} catch (error) {\n\t\t\tthis.telemetry.fetchFailures++;\n\t\t\tif (request.signal?.aborted) throw new WebResearchError(\"ABORTED\", \"Web fetch was aborted\", { cause: error });\n\t\t\tif (timeout.aborted) {\n\t\t\t\tthis.telemetry.timeouts++;\n\t\t\t\tthrow new WebResearchError(\"TIMEOUT\", \"Web fetch timed out\", { cause: error });\n\t\t\t}\n\t\t\tthrow error;\n\t\t}\n\t}\n}\n"]}