{"version":3,"file":"load-web.mjs","names":[],"sources":["../../../../../../../../ai/src/rag/loaders/load-web.ts"],"sourcesContent":["import {\n  guardedFetch,\n  readTextCapped,\n  resolveOutboundPolicy,\n} from \"../../security/outbound-policy\";\nimport { OutboundPolicyError } from \"../../errors\";\nimport type { RagDocument } from \"../contracts/rag-document.type\";\nimport { htmlToText, extractTitle } from \"./load-html\";\nimport type { LoadWebOptions, RagLoaderResult } from \"./loader.type\";\n\n/** Browser-ish UA so servers that gate on it still return prose. */\nconst DEFAULT_USER_AGENT =\n  \"warlock-ai-rag-loader/1.0 (+https://github.com/warlock-js)\";\n\n/**\n * Whether a `content-type` header names an HTML document (so it is run\n * through the tag-strip pass) versus already-plain text (used verbatim).\n */\nfunction isHtmlContentType(contentType: string | undefined): boolean {\n  if (!contentType) {\n    // No header — assume HTML, the common case for a fetched URL.\n    return true;\n  }\n\n  const lower = contentType.toLowerCase();\n\n  return lower.includes(\"text/html\") || lower.includes(\"application/xhtml\");\n}\n\n/**\n * Fetch a URL through the SSRF-safe outbound policy and load it into a single\n * {@link RagDocument} of readable text. The fetch ALWAYS goes through\n * `guardedFetch` — never a raw `fetch` — so the scheme allowlist, host\n * allowlist, post-DNS private-IP guard, timeout, and response-size cap from\n * {@link LoadWebOptions.policy} (or the strict defaults) always apply.\n *\n * HTML responses are run through the same tag-strip pass as {@link loadHtml}\n * (scripts/styles dropped, entities decoded, paragraph structure kept);\n * non-HTML text responses (`text/plain`, markdown, …) are used verbatim.\n * The document's `metadata.source` is the resolved URL, `metadata.title` is\n * the page `<title>` (HTML only, unless overridden), `metadata.contentType`\n * is the server-reported type, and `metadata.loader` is `\"web\"`.\n *\n * The output is the exact shape `index()` consumes, so a load feeds straight\n * in:\n *\n * @example\n * const kb = ai.rag({ embedder, store });\n * await kb.index(await loadWeb(\"https://example.com/guide\"));\n *\n * @example\n * // Tighten the SSRF policy to a single host:\n * await kb.index(await loadWeb(url, {\n *   policy: { hostAllowlist: [\"docs.example.com\"], maxBytes: 2_000_000 },\n *   tags: [\"docs\"],\n * }));\n *\n * @param url - The absolute URL to fetch. Validated by the outbound policy.\n * @param options - `policy` (the {@link OutboundPolicy}) plus shared\n *   `id` / `metadata` / `tags` ({@link LoadWebOptions}).\n * @returns A {@link RagLoaderResult} (one document) ready for `rag.index()`.\n * @throws {OutboundPolicyError} when the policy blocks the URL, the request\n *   times out, the body exceeds the cap, or the response is not OK.\n */\nexport async function loadWeb(\n  url: string,\n  options: LoadWebOptions = {},\n): Promise<RagLoaderResult> {\n  const policy = resolveOutboundPolicy(options.policy);\n\n  const response = await guardedFetch(url, policy, {\n    headers: { \"user-agent\": DEFAULT_USER_AGENT, accept: \"text/html,text/*\" },\n    redirect: \"follow\",\n  });\n\n  if (!response.ok) {\n    throw new OutboundPolicyError(\n      `loadWeb: fetching \"${url}\" returned ${response.status} ${response.statusText}`,\n      { context: { url, status: response.status } },\n    );\n  }\n\n  const contentType = response.headers.get(\"content-type\") ?? undefined;\n  const raw = await readTextCapped(response, policy.maxBytes);\n\n  const isHtml = isHtmlContentType(contentType);\n  const text = isHtml ? htmlToText(raw) : raw.trim();\n  const title = isHtml ? extractTitle(raw) : undefined;\n\n  const id = options.id ?? url;\n\n  // An empty body / all-markup page yields no document, so index() never\n  // receives a no-op record.\n  if (text.length === 0) {\n    return [];\n  }\n\n  // Derived keys sit UNDER the caller's metadata so an explicit override wins.\n  const doc: RagDocument = {\n    id,\n    text,\n    metadata: {\n      source: url,\n      loader: \"web\",\n      ...(title !== undefined ? { title } : {}),\n      ...(contentType !== undefined ? { contentType } : {}),\n      ...options.metadata,\n    },\n    tags: options.tags,\n  };\n\n  return [doc];\n}\n"],"mappings":";;;;;;;AAWA,MAAM,qBACJ;;;;;AAMF,SAAS,kBAAkB,aAA0C;CACnE,IAAI,CAAC,aAEH,OAAO;CAGT,MAAM,QAAQ,YAAY,YAAY;CAEtC,OAAO,MAAM,SAAS,WAAW,KAAK,MAAM,SAAS,mBAAmB;AAC1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqCA,eAAsB,QACpB,KACA,UAA0B,CAAC,GACD;CAC1B,MAAM,SAAS,sBAAsB,QAAQ,MAAM;CAEnD,MAAM,WAAW,MAAM,aAAa,KAAK,QAAQ;EAC/C,SAAS;GAAE,cAAc;GAAoB,QAAQ;EAAmB;EACxE,UAAU;CACZ,CAAC;CAED,IAAI,CAAC,SAAS,IACZ,MAAM,IAAI,oBACR,sBAAsB,IAAI,aAAa,SAAS,OAAO,GAAG,SAAS,cACnE,EAAE,SAAS;EAAE;EAAK,QAAQ,SAAS;CAAO,EAAE,CAC9C;CAGF,MAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;CAC5D,MAAM,MAAM,MAAM,eAAe,UAAU,OAAO,QAAQ;CAE1D,MAAM,SAAS,kBAAkB,WAAW;CAC5C,MAAM,OAAO,SAAS,WAAW,GAAG,IAAI,IAAI,KAAK;CACjD,MAAM,QAAQ,SAAS,aAAa,GAAG,IAAI;CAE3C,MAAM,KAAK,QAAQ,MAAM;CAIzB,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAiBV,OAAO,CAAC;EAZN;EACA;EACA,UAAU;GACR,QAAQ;GACR,QAAQ;GACR,GAAI,UAAU,SAAY,EAAE,MAAM,IAAI,CAAC;GACvC,GAAI,gBAAgB,SAAY,EAAE,YAAY,IAAI,CAAC;GACnD,GAAG,QAAQ;EACb;EACA,MAAM,QAAQ;CAGN,CAAC;AACb"}