` text out of the document, decoded and trimmed, or\n * `undefined` when there is none. Read BEFORE `<head>` is stripped.\n */\nfunction extractTitle(html: string): string | undefined {\n const match = /<title[^>]*>([\\s\\S]*?)<\\/title>/i.exec(html);\n\n if (!match) {\n return undefined;\n }\n\n const title = decodeEntities(match[1]).replace(/\\s+/g, \" \").trim();\n\n return title.length > 0 ? title : undefined;\n}\n\n/**\n * Strip HTML markup down to readable plain text — a lightweight,\n * dependency-free pass (no DOM parser): drop comments and non-prose elements\n * (`script` / `style` / `head` / `svg` / …) content-and-all, convert block\n * tags to line breaks to preserve paragraph structure, remove every\n * remaining tag, decode entities, then collapse runs of whitespace while\n * keeping blank-line paragraph separators.\n */\nfunction htmlToText(html: string): string {\n let text = html;\n\n // 1. Comments first — a commented-out `<script>` must not survive.\n text = text.replace(//g, \" \");\n\n // 2. Non-prose elements, content and all.\n for (const tag of STRIPPED_ELEMENTS) {\n const element = new RegExp(`<${tag}\\\\b[^>]*>[\\\\s\\\\S]*?<\\\\/${tag}>`, \"gi\");\n text = text.replace(element, \" \");\n // Defensively drop a self-closing / unterminated open tag too.\n text = text.replace(new RegExp(`<\\\\/?${tag}\\\\b[^>]*>`, \"gi\"), \" \");\n }\n\n // 3. Block tags → newlines, so paragraph / list structure survives.\n text = text.replace(BLOCK_TAGS, \"\\n\");\n\n // 4. Every remaining tag → gone.\n text = text.replace(/<[^>]+>/g, \"\");\n\n // 5. Entities → characters.\n text = decodeEntities(text);\n\n // 6. Normalize whitespace: trim each line, drop blank runs to a single\n // blank line (a paragraph separator the recursive splitter honors).\n text = text\n .replace(/[^\\S\\n]+/g, \" \")\n .replace(/[ \\t]*\\n[ \\t]*/g, \"\\n\")\n .replace(/\\n{3,}/g, \"\\n\\n\")\n .trim();\n\n return text;\n}\n\n/**\n * Load an HTML string into a single {@link RagDocument} of readable text.\n * Scripts, styles, and other non-prose elements are dropped content-and-all,\n * block tags become line breaks (so paragraph structure survives for the\n * splitter), remaining tags are stripped, and HTML entities are decoded — a\n * lightweight regex pass, no heavy DOM dependency.\n *\n * The document's `metadata.title` is set from the page's `<title>` (unless\n * the caller overrode it), and `metadata.loader` is `\"html\"`. The output is\n * the exact shape `index()` consumes.\n *\n * @example\n * const kb = ai.rag({ embedder, store });\n * await kb.index(loadHtml(rawHtmlString, { id: \"landing-page\" }));\n *\n * @param html - The raw HTML markup.\n * @param options - Shared `id` / `metadata` / `tags` ({@link LoadHtmlOptions}).\n * @returns A {@link RagLoaderResult} (one document) ready for `rag.index()`.\n */\nexport function loadHtml(\n html: string,\n options: LoadHtmlOptions = {},\n): RagLoaderResult {\n const id = options.id ?? DEFAULT_ID;\n const title = extractTitle(html);\n const text = htmlToText(html);\n\n // An all-markup / empty page strips to nothing; emit no document so\n // index() never receives a no-op record (matches loadText's behavior).\n if (text.length === 0) {\n return [];\n }\n\n // Derived keys (source, loader, title) sit UNDER the caller's metadata so\n // an explicit override always wins.\n const doc: RagDocument = {\n id,\n text,\n metadata: {\n source: id,\n loader: \"html\",\n ...(title !== undefined ? { title } : {}),\n ...options.metadata,\n },\n tags: options.tags,\n };\n\n return [doc];\n}\n\n/** Internal — exported for the web loader so it shares the exact strip pass. */\nexport { htmlToText, extractTitle };\n"],"mappings":";;AAIA,MAAM,aAAa;;;;;;;AAQnB,MAAM,oBAAoB;CACxB;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;;AAQA,MAAM,aACJ;;AAGF,MAAM,iBAAyC;CAC7C,KAAK;CACL,IAAI;CACJ,IAAI;CACJ,MAAM;CACN,MAAM;CACN,MAAM;CACN,MAAM;CACN,KAAK;CACL,OAAO;CACP,QAAQ;CACR,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,QAAQ;CACR,MAAM;AACR;;;;;;AAOA,SAAS,eAAe,MAAsB;CAC5C,OAAO,KAAK,QAAQ,sCAAsC,OAAO,SAAiB;EAChF,IAAI,KAAK,OAAO,KAAK;GACnB,MAAM,YACJ,KAAK,OAAO,OAAO,KAAK,OAAO,MAC3B,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE,IACjC,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE;GAEvC,IAAI,OAAO,MAAM,SAAS,KAAK,YAAY,KAAK,YAAY,SAC1D,OAAO;GAGT,IAAI;IACF,OAAO,OAAO,cAAc,SAAS;GACvC,QAAQ;IACN,OAAO;GACT;EACF;EAIA,OAFc,eAAe,KAAK,YAAY,MAE9B;CAClB,CAAC;AACH;;;;;AAMA,SAAS,aAAa,MAAkC;CACtD,MAAM,QAAQ,mCAAmC,KAAK,IAAI;CAE1D,IAAI,CAAC,OACH;CAGF,MAAM,QAAQ,eAAe,MAAM,EAAE,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;CAEjE,OAAO,MAAM,SAAS,IAAI,QAAQ;AACpC;;;;;;;;;AAUA,SAAS,WAAW,MAAsB;CACxC,IAAI,OAAO;CAGX,OAAO,KAAK,QAAQ,oBAAoB,GAAG;CAG3C,KAAK,MAAM,OAAO,mBAAmB;EACnC,MAAM,UAAU,IAAI,OAAO,IAAI,IAAI,yBAAyB,IAAI,IAAI,IAAI;EACxE,OAAO,KAAK,QAAQ,SAAS,GAAG;EAEhC,OAAO,KAAK,QAAQ,IAAI,OAAO,QAAQ,IAAI,YAAY,IAAI,GAAG,GAAG;CACnE;CAGA,OAAO,KAAK,QAAQ,YAAY,IAAI;CAGpC,OAAO,KAAK,QAAQ,YAAY,EAAE;CAGlC,OAAO,eAAe,IAAI;CAI1B,OAAO,KACJ,QAAQ,aAAa,GAAG,CAAC,CACzB,QAAQ,mBAAmB,IAAI,CAAC,CAChC,QAAQ,WAAW,MAAM,CAAC,CAC1B,KAAK;CAER,OAAO;AACT;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,SACd,MACA,UAA2B,CAAC,GACX;CACjB,MAAM,KAAK,QAAQ,MAAM;CACzB,MAAM,QAAQ,aAAa,IAAI;CAC/B,MAAM,OAAO,WAAW,IAAI;CAI5B,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAiBV,OAAO,CAAC;EAXN;EACA;EACA,UAAU;GACR,QAAQ;GACR,QAAQ;GACR,GAAI,UAAU,SAAY,EAAE,MAAM,IAAI,CAAC;GACvC,GAAG,QAAQ;EACb;EACA,MAAM,QAAQ;CAGN,CAAC;AACb"}

{"version":3,"file":"load-html.mjs","names":[],"sources":["../../../../../../../../ai/src/rag/loaders/load-html.ts"],"sourcesContent":["import type { RagDocument } from \"../contracts/rag-document.type\";\nimport type { LoadHtmlOptions, RagLoaderResult } from \"./loader.type\";\n\n/** Default `id` when the caller supplies none. */\nconst DEFAULT_ID = \"document\";\n\n/**\n * Elements whose *content* is not human-readable text and must be removed\n * wholesale (open tag → close tag → everything in between) before tags are\n * stripped. `script` / `style` would otherwise leak code into the chunked\n * text; `noscript` / `template` / `head` / `svg` are non-prose noise.\n */\nconst STRIPPED_ELEMENTS = [\n \"script\",\n \"style\",\n \"noscript\",\n \"template\",\n \"head\",\n \"svg\",\n];\n\n/**\n * Block-level tags that imply a line break in the readable text. Replacing\n * them with `\\n` BEFORE the generic tag strip keeps paragraph / list / table\n * structure (so the recursive splitter still sees `\\n\\n` boundaries) instead\n * of collapsing the whole page onto one line.\n */\nconst BLOCK_TAGS =\n /<\\/?(?:p|div|section|article|header|footer|main|aside|nav|h[1-6]|ul|ol|li|table|tr|td|th|thead|tbody|blockquote|pre|hr|br)\\b[^>]*>/gi;\n\n/** Named HTML entities common in prose. Numeric entities are decoded generically. */\nconst NAMED_ENTITIES: Record = {\n amp: \"&\",\n lt: \"<\",\n gt: \">\",\n quot: '\"',\n apos: \"'\",\n nbsp: \" \",\n copy: \"©\",\n reg: \"®\",\n trade: \"™\",\n hellip: \"…\",\n mdash: \"—\",\n ndash: \"–\",\n lsquo: \"‘\",\n rsquo: \"’\",\n ldquo: \"“\",\n rdquo: \"”\",\n laquo: \"«\",\n raquo: \"»\",\n middot: \"·\",\n bull: \"•\",\n};\n\n/**\n * Decode the HTML entities that survive tag stripping: named (`&`),\n * decimal (`©`), and hex (`©`). Unknown named entities are left\n * verbatim rather than dropped, so unusual markup never silently loses text.\n */\nfunction decodeEntities(text: string): string {\n return text.replace(/&(#x?[0-9a-f]+|[a-z][a-z0-9]*);/gi, (match, body: string) => {\n if (body[0] === \"#\") {\n const codePoint =\n body[1] === \"x\" || body[1] === \"X\"\n ? Number.parseInt(body.slice(2), 16)\n : Number.parseInt(body.slice(1), 10);\n\n if (Number.isNaN(codePoint) || codePoint < 0 || codePoint > 0x10ffff) {\n return match;\n }\n\n try {\n return String.fromCodePoint(codePoint);\n } catch {\n return match;\n }\n }\n\n const named = NAMED_ENTITIES[body.toLowerCase()];\n\n return named ?? match;\n });\n}\n\n/**\n * Pull the `` text out of the document, decoded and trimmed, or\n * `undefined` when there is none. Read BEFORE `<head>` is stripped.\n */\nfunction extractTitle(html: string): string | undefined {\n const match = /<title[^>]*>([\\s\\S]*?)<\\/title>/i.exec(html);\n\n if (!match) {\n return undefined;\n }\n\n const title = decodeEntities(match[1]).replace(/\\s+/g, \" \").trim();\n\n return title.length > 0 ? title : undefined;\n}\n\n/**\n * Strip HTML markup down to readable plain text — a lightweight,\n * dependency-free pass (no DOM parser): drop comments and non-prose elements\n * (`script` / `style` / `head` / `svg` / …) content-and-all, convert block\n * tags to line breaks to preserve paragraph structure, remove every\n * remaining tag, decode entities, then collapse runs of whitespace while\n * keeping blank-line paragraph separators.\n */\nfunction htmlToText(html: string): string {\n let text = html;\n\n // 1. Comments first — a commented-out `<script>` must not survive.\n text = text.replace(//g, \" \");\n\n // 2. Non-prose elements, content and all.\n for (const tag of STRIPPED_ELEMENTS) {\n const element = new RegExp(`<${tag}\\\\b[^>]*>[\\\\s\\\\S]*?<\\\\/${tag}>`, \"gi\");\n text = text.replace(element, \" \");\n // Defensively drop a self-closing / unterminated open tag too.\n text = text.replace(new RegExp(`<\\\\/?${tag}\\\\b[^>]*>`, \"gi\"), \" \");\n }\n\n // 3. Block tags → newlines, so paragraph / list structure survives.\n text = text.replace(BLOCK_TAGS, \"\\n\");\n\n // 4. Every remaining tag → gone.\n text = text.replace(/<[^>]+>/g, \"\");\n\n // 5. Entities → characters.\n text = decodeEntities(text);\n\n // 6. Normalize whitespace: trim each line, drop blank runs to a single\n // blank line (a paragraph separator the recursive splitter honors).\n text = text\n .replace(/[^\\S\\n]+/g, \" \")\n .replace(/[ \\t]*\\n[ \\t]*/g, \"\\n\")\n .replace(/\\n{3,}/g, \"\\n\\n\")\n .trim();\n\n return text;\n}\n\n/**\n * Load an HTML string into a single {@link RagDocument} of readable text.\n * Scripts, styles, and other non-prose elements are dropped content-and-all,\n * block tags become line breaks (so paragraph structure survives for the\n * splitter), remaining tags are stripped, and HTML entities are decoded — a\n * lightweight regex pass, no heavy DOM dependency.\n *\n * The document's `metadata.title` is set from the page's `<title>` (unless\n * the caller overrode it), and `metadata.loader` is `\"html\"`. The output is\n * the exact shape `index()` consumes.\n *\n * @example\n * const kb = ai.rag({ embedder, store });\n * await kb.index(loadHtml(rawHtmlString, { id: \"landing-page\" }));\n *\n * @param html - The raw HTML markup.\n * @param options - Shared `id` / `metadata` / `tags` ({@link LoadHtmlOptions}).\n * @returns A {@link RagLoaderResult} (one document) ready for `rag.index()`.\n */\nexport function loadHtml(\n html: string,\n options: LoadHtmlOptions = {},\n): RagLoaderResult {\n const id = options.id ?? DEFAULT_ID;\n const title = extractTitle(html);\n const text = htmlToText(html);\n\n // An all-markup / empty page strips to nothing; emit no document so\n // index() never receives a no-op record (matches loadText's behavior).\n if (text.length === 0) {\n return [];\n }\n\n // Derived keys (source, loader, title) sit UNDER the caller's metadata so\n // an explicit override always wins.\n const doc: RagDocument = {\n id,\n text,\n metadata: {\n source: id,\n loader: \"html\",\n ...(title !== undefined ? { title } : {}),\n ...options.metadata,\n },\n tags: options.tags,\n };\n\n return [doc];\n}\n\n/** Internal — exported for the web loader so it shares the exact strip pass. */\nexport { htmlToText, extractTitle };\n"],"mappings":";;AAIA,MAAM,aAAa;;;;;;;AAQnB,MAAM,oBAAoB;CACxB;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;;AAQA,MAAM,aACJ;;AAGF,MAAM,iBAAyC;CAC7C,KAAK;CACL,IAAI;CACJ,IAAI;CACJ,MAAM;CACN,MAAM;CACN,MAAM;CACN,MAAM;CACN,KAAK;CACL,OAAO;CACP,QAAQ;CACR,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,QAAQ;CACR,MAAM;AACR;;;;;;AAOA,SAAS,eAAe,MAAsB;CAC5C,OAAO,KAAK,QAAQ,sCAAsC,OAAO,SAAiB;EAChF,IAAI,KAAK,OAAO,KAAK;GACnB,MAAM,YACJ,KAAK,OAAO,OAAO,KAAK,OAAO,MAC3B,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE,IACjC,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE;GAEvC,IAAI,OAAO,MAAM,SAAS,KAAK,YAAY,KAAK,YAAY,SAC1D,OAAO;GAGT,IAAI;IACF,OAAO,OAAO,cAAc,SAAS;GACvC,QAAQ;IACN,OAAO;GACT;EACF;EAIA,OAFc,eAAe,KAAK,YAAY,MAE9B;CAClB,CAAC;AACH;;;;;AAMA,SAAS,aAAa,MAAkC;CACtD,MAAM,QAAQ,mCAAmC,KAAK,IAAI;CAE1D,IAAI,CAAC,OACH;CAGF,MAAM,QAAQ,eAAe,MAAM,EAAE,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;CAEjE,OAAO,MAAM,SAAS,IAAI,QAAQ;AACpC;;;;;;;;;AAUA,SAAS,WAAW,MAAsB;CACxC,IAAI,OAAO;CAGX,OAAO,KAAK,QAAQ,oBAAoB,GAAG;CAG3C,KAAK,MAAM,OAAO,mBAAmB;EACnC,MAAM,UAAU,IAAI,OAAO,IAAI,IAAI,yBAAyB,IAAI,IAAI,IAAI;EACxE,OAAO,KAAK,QAAQ,SAAS,GAAG;EAEhC,OAAO,KAAK,QAAQ,IAAI,OAAO,QAAQ,IAAI,YAAY,IAAI,GAAG,GAAG;CACnE;CAGA,OAAO,KAAK,QAAQ,YAAY,IAAI;CAGpC,OAAO,KAAK,QAAQ,YAAY,EAAE;CAGlC,OAAO,eAAe,IAAI;CAI1B,OAAO,KACJ,QAAQ,aAAa,GAAG,CAAC,CACzB,QAAQ,mBAAmB,IAAI,CAAC,CAChC,QAAQ,WAAW,MAAM,CAAC,CAC1B,KAAK;CAER,OAAO;AACT;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,SACd,MACA,UAA2B,CAAC,GACX;CACjB,MAAM,KAAK,QAAQ,MAAM;CACzB,MAAM,QAAQ,aAAa,IAAI;CAC/B,MAAM,OAAO,WAAW,IAAI;CAI5B,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAiBV,OAAO,CAAC;EAXN;EACA;EACA,UAAU;GACR,QAAQ;GACR,QAAQ;GACR,GAAI,UAAU,SAAY,EAAE,MAAM,IAAI,CAAC;GACvC,GAAG,QAAQ;EACb;EACA,MAAM,QAAQ;CAGN,CAAC;AACb"}