{"version":3,"file":"markdown.mjs","names":[],"sources":["../../../../../../../../ai/src/rag/chunk/markdown.ts"],"sourcesContent":["import type { Chunk } from \"../contracts/chunk-options.type\";\nimport { DEFAULT_SEPARATORS, recursiveChunk } from \"./recursive\";\n\n/** Matches an ATX Markdown heading line (`#` … `######`) at line start. */\nconst HEADING_LINE = /^#{1,6}[ \\t].*$/gm;\n\n/**\n * Markdown heading/section-aware splitter.\n *\n * Splits the document on ATX heading boundaries (`#`…`######`) first so a\n * section's heading stays glued to its body, then applies the recursive\n * character splitter WITHIN each section so any section larger than `size`\n * is broken down further. Sections at or under `size` are emitted whole.\n * Spans are exact relative to the original document.\n */\nexport function markdownChunk(\n  text: string,\n  size: number,\n  overlap: number,\n  separators: string[] = DEFAULT_SEPARATORS,\n): Chunk[] {\n  if (text.length === 0) {\n    return [];\n  }\n\n  const sections = splitSections(text);\n  const chunks: Chunk[] = [];\n  let index = 0;\n\n  for (const section of sections) {\n    const body = text.slice(section.start, section.end);\n\n    if (body.trim().length === 0) {\n      continue;\n    }\n\n    if (body.length <= size) {\n      chunks.push({\n        text: body,\n        index,\n        span: [section.start, section.end],\n      });\n      index += 1;\n\n      continue;\n    }\n\n    // Recurse within the section, then shift the relative spans to\n    // absolute document offsets and renumber sequentially.\n    const inner = recursiveChunk(body, size, overlap, separators);\n\n    for (const piece of inner) {\n      chunks.push({\n        text: piece.text,\n        index,\n        span: [section.start + piece.span[0], section.start + piece.span[1]],\n      });\n      index += 1;\n    }\n  }\n\n  return chunks;\n}\n\n/** A section's absolute `[start, end)` span (heading line + body until next heading). */\ntype SectionSpan = {\n  start: number;\n  end: number;\n};\n\n/**\n * Carve the document into sections, each beginning at a heading line and\n * running until the next heading (the preamble before the first heading is\n * its own section). Spans cover the whole document with no gaps.\n */\nfunction splitSections(text: string): SectionSpan[] {\n  const starts: number[] = [];\n  let match: RegExpExecArray | null;\n\n  HEADING_LINE.lastIndex = 0;\n\n  while ((match = HEADING_LINE.exec(text)) !== null) {\n    starts.push(match.index);\n  }\n\n  // No headings at all — the whole document is one section.\n  if (starts.length === 0) {\n    return [{ start: 0, end: text.length }];\n  }\n\n  const sections: SectionSpan[] = [];\n\n  // Preamble before the first heading, if any.\n  if (starts[0] > 0) {\n    sections.push({ start: 0, end: starts[0] });\n  }\n\n  starts.forEach((start, position) => {\n    const end = position + 1 < starts.length ? starts[position + 1] : text.length;\n\n    sections.push({ start, end });\n  });\n\n  return sections;\n}\n"],"mappings":";;;;AAIA,MAAM,eAAe;;;;;;;;;;AAWrB,SAAgB,cACd,MACA,MACA,SACA,aAAuB,oBACd;CACT,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAGV,MAAM,WAAW,cAAc,IAAI;CACnC,MAAM,SAAkB,CAAC;CACzB,IAAI,QAAQ;CAEZ,KAAK,MAAM,WAAW,UAAU;EAC9B,MAAM,OAAO,KAAK,MAAM,QAAQ,OAAO,QAAQ,GAAG;EAElD,IAAI,KAAK,KAAK,CAAC,CAAC,WAAW,GACzB;EAGF,IAAI,KAAK,UAAU,MAAM;GACvB,OAAO,KAAK;IACV,MAAM;IACN;IACA,MAAM,CAAC,QAAQ,OAAO,QAAQ,GAAG;GACnC,CAAC;GACD,SAAS;GAET;EACF;EAIA,MAAM,QAAQ,eAAe,MAAM,MAAM,SAAS,UAAU;EAE5D,KAAK,MAAM,SAAS,OAAO;GACzB,OAAO,KAAK;IACV,MAAM,MAAM;IACZ;IACA,MAAM,CAAC,QAAQ,QAAQ,MAAM,KAAK,IAAI,QAAQ,QAAQ,MAAM,KAAK,EAAE;GACrE,CAAC;GACD,SAAS;EACX;CACF;CAEA,OAAO;AACT;;;;;;AAaA,SAAS,cAAc,MAA6B;CAClD,MAAM,SAAmB,CAAC;CAC1B,IAAI;CAEJ,aAAa,YAAY;CAEzB,QAAQ,QAAQ,aAAa,KAAK,IAAI,OAAO,MAC3C,OAAO,KAAK,MAAM,KAAK;CAIzB,IAAI,OAAO,WAAW,GACpB,OAAO,CAAC;EAAE,OAAO;EAAG,KAAK,KAAK;CAAO,CAAC;CAGxC,MAAM,WAA0B,CAAC;CAGjC,IAAI,OAAO,KAAK,GACd,SAAS,KAAK;EAAE,OAAO;EAAG,KAAK,OAAO;CAAG,CAAC;CAG5C,OAAO,SAAS,OAAO,aAAa;EAClC,MAAM,MAAM,WAAW,IAAI,OAAO,SAAS,OAAO,WAAW,KAAK,KAAK;EAEvE,SAAS,KAAK;GAAE;GAAO;EAAI,CAAC;CAC9B,CAAC;CAED,OAAO;AACT"}