{"version":3,"file":"recursive.mjs","names":[],"sources":["../../../../../../../../ai/src/rag/chunk/recursive.ts"],"sourcesContent":["import type { Chunk } from \"../contracts/chunk-options.type\";\n\n/** Default separators for the recursive splitter, tried largest-unit first. */\nexport const DEFAULT_SEPARATORS: string[] = [\"\\n\\n\", \"\\n\", \". \", \" \", \"\"];\n\n/**\n * Recursive character splitter (the default strategy).\n *\n * Walks `separators` largest-unit-first: it splits the text on the first\n * separator, then packs the resulting pieces into chunks up to `size`\n * characters, carrying `overlap` characters forward between adjacent\n * chunks. Any single piece that is itself larger than `size` is split\n * again on the next-finer separator, recursing until a piece fits (the\n * `\"\"` separator is the final char-by-char fallback).\n *\n * Every emitted chunk records its exact `[start, end)` character span in\n * the ORIGINAL text so a citation can point back precisely — spans are\n * tracked by index-of search as packed pieces are joined.\n *\n * Character-based and deliberately tokenizer-free.\n */\nexport function recursiveChunk(\n  text: string,\n  size: number,\n  overlap: number,\n  separators: string[] = DEFAULT_SEPARATORS,\n): Chunk[] {\n  const pieces = splitToPieces(text, size, separators);\n\n  // Re-anchor each packed piece to its absolute offset in `text`. Pieces\n  // are non-overlapping and in document order, so a forward cursor finds\n  // each one's true start even when the same substring repeats.\n  const spans = anchorPieces(text, pieces);\n\n  return packPieces(text, spans, size, overlap);\n}\n\n/**\n * Recursively split `text` into pieces no larger than `size` using the\n * ordered separator list. Pieces preserve original characters (no\n * trimming) so downstream span anchoring stays exact.\n */\nfunction splitToPieces(text: string, size: number, separators: string[]): string[] {\n  if (text.length <= size) {\n    return text.length > 0 ? [text] : [];\n  }\n\n  const [separator, ...rest] = separators;\n\n  // Exhausted every separator (or hit the char fallback) — hard-split by\n  // size so an oversize unit never blows the budget.\n  if (separator === undefined || separator === \"\") {\n    return hardSplit(text, size);\n  }\n\n  const segments = splitKeepingSeparator(text, separator);\n  const pieces: string[] = [];\n\n  for (const segment of segments) {\n    if (segment.length === 0) {\n      continue;\n    }\n\n    if (segment.length <= size) {\n      pieces.push(segment);\n\n      continue;\n    }\n\n    pieces.push(...splitToPieces(segment, size, rest));\n  }\n\n  return pieces;\n}\n\n/**\n * Split on `separator` but re-attach the separator to the end of each\n * preceding segment, so concatenating the segments reconstructs the\n * original text verbatim (keeping spans exact).\n */\nfunction splitKeepingSeparator(text: string, separator: string): string[] {\n  const raw = text.split(separator);\n  const segments: string[] = [];\n\n  raw.forEach((part, position) => {\n    const isLast = position === raw.length - 1;\n\n    segments.push(isLast ? part : part + separator);\n  });\n\n  return segments;\n}\n\n/** Hard char-window split for a unit larger than `size` with no usable separator. */\nfunction hardSplit(text: string, size: number): string[] {\n  const pieces: string[] = [];\n\n  for (let cursor = 0; cursor < text.length; cursor += size) {\n    pieces.push(text.slice(cursor, cursor + size));\n  }\n\n  return pieces;\n}\n\n/** A piece plus its absolute `[start, end)` span in the original text. */\ntype AnchoredPiece = {\n  text: string;\n  start: number;\n  end: number;\n};\n\n/**\n * Map each piece back to its absolute offset using a monotonic cursor —\n * pieces are emitted in document order, so the next occurrence at-or-after\n * the cursor is the correct one even for repeated substrings.\n */\nfunction anchorPieces(text: string, pieces: string[]): AnchoredPiece[] {\n  const anchored: AnchoredPiece[] = [];\n  let cursor = 0;\n\n  for (const piece of pieces) {\n    const start = text.indexOf(piece, cursor);\n    const resolvedStart = start === -1 ? cursor : start;\n    const end = resolvedStart + piece.length;\n\n    anchored.push({ text: piece, start: resolvedStart, end });\n    cursor = end;\n  }\n\n  return anchored;\n}\n\n/**\n * Greedily pack anchored pieces into chunks up to `size` characters, then\n * carry `overlap` trailing characters from each emitted chunk into the\n * next so context is not lost at a boundary. Spans are taken straight\n * from the anchored pieces, so the overlap text is part of the next\n * chunk's span exactly.\n */\nfunction packPieces(\n  text: string,\n  pieces: AnchoredPiece[],\n  size: number,\n  overlap: number,\n): Chunk[] {\n  const chunks: Chunk[] = [];\n\n  let bufferStart = -1;\n  let bufferEnd = -1;\n  let index = 0;\n\n  const flush = (): void => {\n    if (bufferStart === -1) {\n      return;\n    }\n\n    chunks.push({\n      text: text.slice(bufferStart, bufferEnd),\n      index,\n      span: [bufferStart, bufferEnd],\n    });\n    index += 1;\n  };\n\n  for (const piece of pieces) {\n    if (bufferStart === -1) {\n      bufferStart = piece.start;\n      bufferEnd = piece.end;\n\n      continue;\n    }\n\n    const projected = piece.end - bufferStart;\n\n    if (projected <= size) {\n      bufferEnd = piece.end;\n\n      continue;\n    }\n\n    flush();\n\n    // Start the next buffer `overlap` chars before this piece (clamped to\n    // the previous chunk's start) so adjacent chunks share context.\n    const overlapStart = overlap > 0 ? Math.max(bufferStart, piece.start - overlap) : piece.start;\n\n    bufferStart = overlapStart;\n    bufferEnd = piece.end;\n  }\n\n  flush();\n\n  return chunks;\n}\n"],"mappings":";;AAGA,MAAa,qBAA+B;CAAC;CAAQ;CAAM;CAAM;CAAK;AAAE;;;;;;;;;;;;;;;;;AAkBxE,SAAgB,eACd,MACA,MACA,SACA,aAAuB,oBACd;CAQT,OAAO,WAAW,MAFJ,aAAa,MALZ,cAAc,MAAM,MAAM,UAKH,CAEV,GAAG,MAAM,OAAO;AAC9C;;;;;;AAOA,SAAS,cAAc,MAAc,MAAc,YAAgC;CACjF,IAAI,KAAK,UAAU,MACjB,OAAO,KAAK,SAAS,IAAI,CAAC,IAAI,IAAI,CAAC;CAGrC,MAAM,CAAC,WAAW,GAAG,QAAQ;CAI7B,IAAI,cAAc,UAAa,cAAc,IAC3C,OAAO,UAAU,MAAM,IAAI;CAG7B,MAAM,WAAW,sBAAsB,MAAM,SAAS;CACtD,MAAM,SAAmB,CAAC;CAE1B,KAAK,MAAM,WAAW,UAAU;EAC9B,IAAI,QAAQ,WAAW,GACrB;EAGF,IAAI,QAAQ,UAAU,MAAM;GAC1B,OAAO,KAAK,OAAO;GAEnB;EACF;EAEA,OAAO,KAAK,GAAG,cAAc,SAAS,MAAM,IAAI,CAAC;CACnD;CAEA,OAAO;AACT;;;;;;AAOA,SAAS,sBAAsB,MAAc,WAA6B;CACxE,MAAM,MAAM,KAAK,MAAM,SAAS;CAChC,MAAM,WAAqB,CAAC;CAE5B,IAAI,SAAS,MAAM,aAAa;EAC9B,MAAM,SAAS,aAAa,IAAI,SAAS;EAEzC,SAAS,KAAK,SAAS,OAAO,OAAO,SAAS;CAChD,CAAC;CAED,OAAO;AACT;;AAGA,SAAS,UAAU,MAAc,MAAwB;CACvD,MAAM,SAAmB,CAAC;CAE1B,KAAK,IAAI,SAAS,GAAG,SAAS,KAAK,QAAQ,UAAU,MACnD,OAAO,KAAK,KAAK,MAAM,QAAQ,SAAS,IAAI,CAAC;CAG/C,OAAO;AACT;;;;;;AAcA,SAAS,aAAa,MAAc,QAAmC;CACrE,MAAM,WAA4B,CAAC;CACnC,IAAI,SAAS;CAEb,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,QAAQ,KAAK,QAAQ,OAAO,MAAM;EACxC,MAAM,gBAAgB,UAAU,KAAK,SAAS;EAC9C,MAAM,MAAM,gBAAgB,MAAM;EAElC,SAAS,KAAK;GAAE,MAAM;GAAO,OAAO;GAAe;EAAI,CAAC;EACxD,SAAS;CACX;CAEA,OAAO;AACT;;;;;;;;AASA,SAAS,WACP,MACA,QACA,MACA,SACS;CACT,MAAM,SAAkB,CAAC;CAEzB,IAAI,cAAc;CAClB,IAAI,YAAY;CAChB,IAAI,QAAQ;CAEZ,MAAM,cAAoB;EACxB,IAAI,gBAAgB,IAClB;EAGF,OAAO,KAAK;GACV,MAAM,KAAK,MAAM,aAAa,SAAS;GACvC;GACA,MAAM,CAAC,aAAa,SAAS;EAC/B,CAAC;EACD,SAAS;CACX;CAEA,KAAK,MAAM,SAAS,QAAQ;EAC1B,IAAI,gBAAgB,IAAI;GACtB,cAAc,MAAM;GACpB,YAAY,MAAM;GAElB;EACF;EAIA,IAFkB,MAAM,MAAM,eAEb,MAAM;GACrB,YAAY,MAAM;GAElB;EACF;EAEA,MAAM;EAMN,cAFqB,UAAU,IAAI,KAAK,IAAI,aAAa,MAAM,QAAQ,OAAO,IAAI,MAAM;EAGxF,YAAY,MAAM;CACpB;CAEA,MAAM;CAEN,OAAO;AACT"}