{"version":3,"file":"text_splitter.cjs","names":["BaseDocumentTransformer","fields?: Partial<TextSplitterParams>","text: string","documents: Document[]","chunkHeaderOptions: TextSplitterChunkHeaderOptions","separator: string","texts: string[]","metadatas: Record<string, any>[]","_metadatas: Record<string, any>[]","Document","start?: number","end?: number","docs: string[]","splits: string[]","currentDoc: string[]","doc","fields?: Partial<CharacterTextSplitterParams>","fields?: Partial<RecursiveCharacterTextSplitterParams>","separators: string[]","finalChunks: string[]","goodSplits: string[]","language: SupportedTextSplitterLanguage","options?: Partial<RecursiveCharacterTextSplitterParams>","fields?: Partial<TokenTextSplitterParams>","fields?: Partial<MarkdownTextSplitterParams>","fields?: Partial<LatexTextSplitterParams>"],"sources":["../src/text_splitter.ts"],"sourcesContent":["import type * as tiktoken from \"js-tiktoken\";\nimport { Document, BaseDocumentTransformer } from \"@langchain/core/documents\";\nimport { getEncoding } from \"@langchain/core/utils/tiktoken\";\n\nexport interface TextSplitterParams {\n  chunkSize: number;\n  chunkOverlap: number;\n  keepSeparator: boolean;\n  lengthFunction?:\n    | ((text: string) => number)\n    | ((text: string) => Promise<number>);\n}\n\nexport type TextSplitterChunkHeaderOptions = {\n  chunkHeader?: string;\n  chunkOverlapHeader?: string;\n  appendChunkOverlapHeader?: boolean;\n};\n\nexport abstract class TextSplitter\n  extends BaseDocumentTransformer\n  implements TextSplitterParams\n{\n  lc_namespace = [\"langchain\", \"document_transformers\", \"text_splitters\"];\n\n  chunkSize = 1000;\n\n  chunkOverlap = 200;\n\n  keepSeparator = false;\n\n  lengthFunction:\n    | ((text: string) => number)\n    | ((text: string) => Promise<number>);\n\n  constructor(fields?: Partial<TextSplitterParams>) {\n    super(fields);\n    this.chunkSize = fields?.chunkSize ?? this.chunkSize;\n    this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap;\n    this.keepSeparator = fields?.keepSeparator ?? this.keepSeparator;\n    this.lengthFunction =\n      fields?.lengthFunction ?? ((text: string) => text.length);\n    if (this.chunkOverlap >= this.chunkSize) {\n      throw new Error(\"Cannot have chunkOverlap >= chunkSize\");\n    }\n  }\n\n  async transformDocuments(\n    documents: Document[],\n    chunkHeaderOptions: TextSplitterChunkHeaderOptions = {}\n  ): Promise<Document[]> {\n    return this.splitDocuments(documents, chunkHeaderOptions);\n  }\n\n  abstract splitText(text: string): Promise<string[]>;\n\n  protected splitOnSeparator(text: string, separator: string): string[] {\n    let splits;\n    if (separator) {\n      if (this.keepSeparator) {\n        const regexEscapedSeparator = separator.replace(\n          /[/\\-\\\\^$*+?.()|[\\]{}]/g,\n          \"\\\\$&\"\n        );\n        splits = text.split(new RegExp(`(?=${regexEscapedSeparator})`));\n      } else {\n        splits = text.split(separator);\n      }\n    } else {\n      splits = text.split(\"\");\n    }\n    return splits.filter((s) => s !== \"\");\n  }\n\n  async createDocuments(\n    texts: string[],\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    metadatas: Record<string, any>[] = [],\n    chunkHeaderOptions: TextSplitterChunkHeaderOptions = {}\n  ): Promise<Document[]> {\n    // if no metadata is provided, we create an empty one for each text\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    const _metadatas: Record<string, any>[] =\n      metadatas.length > 0\n        ? metadatas\n        : [...Array(texts.length)].map(() => ({}));\n    const {\n      chunkHeader = \"\",\n      chunkOverlapHeader = \"(cont'd) \",\n      appendChunkOverlapHeader = false,\n    } = chunkHeaderOptions;\n    const documents = new Array<Document>();\n    for (let i = 0; i < texts.length; i += 1) {\n      const text = texts[i];\n      let lineCounterIndex = 1;\n      let prevChunk = null;\n      let indexPrevChunk = -1;\n      for (const chunk of await this.splitText(text)) {\n        let pageContent = chunkHeader;\n\n        // we need to count the \\n that are in the text before getting removed by the splitting\n        const indexChunk = text.indexOf(chunk, indexPrevChunk + 1);\n        if (prevChunk === null) {\n          const newLinesBeforeFirstChunk = this.numberOfNewLines(\n            text,\n            0,\n            indexChunk\n          );\n          lineCounterIndex += newLinesBeforeFirstChunk;\n        } else {\n          const indexEndPrevChunk =\n            indexPrevChunk + (await this.lengthFunction(prevChunk));\n          if (indexEndPrevChunk < indexChunk) {\n            const numberOfIntermediateNewLines = this.numberOfNewLines(\n              text,\n              indexEndPrevChunk,\n              indexChunk\n            );\n            lineCounterIndex += numberOfIntermediateNewLines;\n          } else if (indexEndPrevChunk > indexChunk) {\n            const numberOfIntermediateNewLines = this.numberOfNewLines(\n              text,\n              indexChunk,\n              indexEndPrevChunk\n            );\n            lineCounterIndex -= numberOfIntermediateNewLines;\n          }\n          if (appendChunkOverlapHeader) {\n            pageContent += chunkOverlapHeader;\n          }\n        }\n        const newLinesCount = this.numberOfNewLines(chunk);\n\n        const loc =\n          _metadatas[i].loc && typeof _metadatas[i].loc === \"object\"\n            ? { ..._metadatas[i].loc }\n            : {};\n        loc.lines = {\n          from: lineCounterIndex,\n          to: lineCounterIndex + newLinesCount,\n        };\n        const metadataWithLinesNumber = {\n          ..._metadatas[i],\n          loc,\n        };\n\n        pageContent += chunk;\n        documents.push(\n          new Document({\n            pageContent,\n            metadata: metadataWithLinesNumber,\n          })\n        );\n        lineCounterIndex += newLinesCount;\n        prevChunk = chunk;\n        indexPrevChunk = indexChunk;\n      }\n    }\n    return documents;\n  }\n\n  private numberOfNewLines(text: string, start?: number, end?: number) {\n    const textSection = text.slice(start, end);\n    return (textSection.match(/\\n/g) || []).length;\n  }\n\n  async splitDocuments(\n    documents: Document[],\n    chunkHeaderOptions: TextSplitterChunkHeaderOptions = {}\n  ): Promise<Document[]> {\n    const selectedDocuments = documents.filter(\n      (doc) => doc.pageContent !== undefined\n    );\n    const texts = selectedDocuments.map((doc) => doc.pageContent);\n    const metadatas = selectedDocuments.map((doc) => doc.metadata);\n    return this.createDocuments(texts, metadatas, chunkHeaderOptions);\n  }\n\n  private joinDocs(docs: string[], separator: string): string | null {\n    const text = docs.join(separator).trim();\n    return text === \"\" ? null : text;\n  }\n\n  async mergeSplits(splits: string[], separator: string): Promise<string[]> {\n    const docs: string[] = [];\n    const currentDoc: string[] = [];\n    let total = 0;\n    for (const d of splits) {\n      const _len = await this.lengthFunction(d);\n      if (\n        total + _len + currentDoc.length * separator.length >\n        this.chunkSize\n      ) {\n        if (total > this.chunkSize) {\n          console.warn(\n            `Created a chunk of size ${total}, +\nwhich is longer than the specified ${this.chunkSize}`\n          );\n        }\n        if (currentDoc.length > 0) {\n          const doc = this.joinDocs(currentDoc, separator);\n          if (doc !== null) {\n            docs.push(doc);\n          }\n          // Keep on popping if:\n          // - we have a larger chunk than in the chunk overlap\n          // - or if we still have any chunks and the length is long\n          while (\n            total > this.chunkOverlap ||\n            (total + _len + currentDoc.length * separator.length >\n              this.chunkSize &&\n              total > 0)\n          ) {\n            total -= await this.lengthFunction(currentDoc[0]);\n            currentDoc.shift();\n          }\n        }\n      }\n      currentDoc.push(d);\n      total += _len;\n    }\n    const doc = this.joinDocs(currentDoc, separator);\n    if (doc !== null) {\n      docs.push(doc);\n    }\n    return docs;\n  }\n}\n\nexport interface CharacterTextSplitterParams extends TextSplitterParams {\n  separator: string;\n}\n\nexport class CharacterTextSplitter\n  extends TextSplitter\n  implements CharacterTextSplitterParams\n{\n  static lc_name() {\n    return \"CharacterTextSplitter\";\n  }\n\n  separator = \"\\n\\n\";\n\n  constructor(fields?: Partial<CharacterTextSplitterParams>) {\n    super(fields);\n    this.separator = fields?.separator ?? this.separator;\n  }\n\n  async splitText(text: string): Promise<string[]> {\n    // First we naively split the large input into a bunch of smaller ones.\n    const splits = this.splitOnSeparator(text, this.separator);\n    return this.mergeSplits(splits, this.keepSeparator ? \"\" : this.separator);\n  }\n}\n\nexport interface RecursiveCharacterTextSplitterParams\n  extends TextSplitterParams {\n  separators: string[];\n}\n\nexport const SupportedTextSplitterLanguages = [\n  \"cpp\",\n  \"go\",\n  \"java\",\n  \"js\",\n  \"php\",\n  \"proto\",\n  \"python\",\n  \"rst\",\n  \"ruby\",\n  \"rust\",\n  \"scala\",\n  \"swift\",\n  \"markdown\",\n  \"latex\",\n  \"html\",\n  \"sol\",\n] as const;\n\nexport type SupportedTextSplitterLanguage =\n  (typeof SupportedTextSplitterLanguages)[number];\n\nexport class RecursiveCharacterTextSplitter\n  extends TextSplitter\n  implements RecursiveCharacterTextSplitterParams\n{\n  static lc_name() {\n    return \"RecursiveCharacterTextSplitter\";\n  }\n\n  separators: string[] = [\"\\n\\n\", \"\\n\", \" \", \"\"];\n\n  constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>) {\n    super(fields);\n    this.separators = fields?.separators ?? this.separators;\n    this.keepSeparator = fields?.keepSeparator ?? true;\n  }\n\n  private async _splitText(text: string, separators: string[]) {\n    const finalChunks: string[] = [];\n\n    // Get appropriate separator to use\n    let separator: string = separators[separators.length - 1];\n    let newSeparators;\n    for (let i = 0; i < separators.length; i += 1) {\n      const s = separators[i];\n      if (s === \"\") {\n        separator = s;\n        break;\n      }\n      if (text.includes(s)) {\n        separator = s;\n        newSeparators = separators.slice(i + 1);\n        break;\n      }\n    }\n\n    // Now that we have the separator, split the text\n    const splits = this.splitOnSeparator(text, separator);\n\n    // Now go merging things, recursively splitting longer texts.\n    let goodSplits: string[] = [];\n    const _separator = this.keepSeparator ? \"\" : separator;\n    for (const s of splits) {\n      if ((await this.lengthFunction(s)) < this.chunkSize) {\n        goodSplits.push(s);\n      } else {\n        if (goodSplits.length) {\n          const mergedText = await this.mergeSplits(goodSplits, _separator);\n          finalChunks.push(...mergedText);\n          goodSplits = [];\n        }\n        if (!newSeparators) {\n          finalChunks.push(s);\n        } else {\n          const otherInfo = await this._splitText(s, newSeparators);\n          finalChunks.push(...otherInfo);\n        }\n      }\n    }\n    if (goodSplits.length) {\n      const mergedText = await this.mergeSplits(goodSplits, _separator);\n      finalChunks.push(...mergedText);\n    }\n    return finalChunks;\n  }\n\n  async splitText(text: string): Promise<string[]> {\n    return this._splitText(text, this.separators);\n  }\n\n  static fromLanguage(\n    language: SupportedTextSplitterLanguage,\n    options?: Partial<RecursiveCharacterTextSplitterParams>\n  ) {\n    return new RecursiveCharacterTextSplitter({\n      ...options,\n      separators:\n        RecursiveCharacterTextSplitter.getSeparatorsForLanguage(language),\n    });\n  }\n\n  static getSeparatorsForLanguage(language: SupportedTextSplitterLanguage) {\n    if (language === \"cpp\") {\n      return [\n        // Split along class definitions\n        \"\\nclass \",\n        // Split along function definitions\n        \"\\nvoid \",\n        \"\\nint \",\n        \"\\nfloat \",\n        \"\\ndouble \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"go\") {\n      return [\n        // Split along function definitions\n        \"\\nfunc \",\n        \"\\nvar \",\n        \"\\nconst \",\n        \"\\ntype \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"java\") {\n      return [\n        // Split along class definitions\n        \"\\nclass \",\n        // Split along method definitions\n        \"\\npublic \",\n        \"\\nprotected \",\n        \"\\nprivate \",\n        \"\\nstatic \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"js\") {\n      return [\n        // Split along function definitions\n        \"\\nfunction \",\n        \"\\nconst \",\n        \"\\nlet \",\n        \"\\nvar \",\n        \"\\nclass \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        \"\\ndefault \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"php\") {\n      return [\n        // Split along function definitions\n        \"\\nfunction \",\n        // Split along class definitions\n        \"\\nclass \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nforeach \",\n        \"\\nwhile \",\n        \"\\ndo \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"proto\") {\n      return [\n        // Split along message definitions\n        \"\\nmessage \",\n        // Split along service definitions\n        \"\\nservice \",\n        // Split along enum definitions\n        \"\\nenum \",\n        // Split along option definitions\n        \"\\noption \",\n        // Split along import statements\n        \"\\nimport \",\n        // Split along syntax declarations\n        \"\\nsyntax \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"python\") {\n      return [\n        // First, try to split along class definitions\n        \"\\nclass \",\n        \"\\ndef \",\n        \"\\n\\tdef \",\n        // Now split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"rst\") {\n      return [\n        // Split along section titles\n        \"\\n===\\n\",\n        \"\\n---\\n\",\n        \"\\n***\\n\",\n        // Split along directive markers\n        \"\\n.. \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"ruby\") {\n      return [\n        // Split along method definitions\n        \"\\ndef \",\n        \"\\nclass \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nunless \",\n        \"\\nwhile \",\n        \"\\nfor \",\n        \"\\ndo \",\n        \"\\nbegin \",\n        \"\\nrescue \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"rust\") {\n      return [\n        // Split along function definitions\n        \"\\nfn \",\n        \"\\nconst \",\n        \"\\nlet \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nwhile \",\n        \"\\nfor \",\n        \"\\nloop \",\n        \"\\nmatch \",\n        \"\\nconst \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"scala\") {\n      return [\n        // Split along class definitions\n        \"\\nclass \",\n        \"\\nobject \",\n        // Split along method definitions\n        \"\\ndef \",\n        \"\\nval \",\n        \"\\nvar \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nmatch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"swift\") {\n      return [\n        // Split along function definitions\n        \"\\nfunc \",\n        // Split along class definitions\n        \"\\nclass \",\n        \"\\nstruct \",\n        \"\\nenum \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\ndo \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"markdown\") {\n      return [\n        // First, try to split along Markdown headings (starting with level 2)\n        \"\\n## \",\n        \"\\n### \",\n        \"\\n#### \",\n        \"\\n##### \",\n        \"\\n###### \",\n        // Note the alternative syntax for headings (below) is not handled here\n        // Heading level 2\n        // ---------------\n        // End of code block\n        \"```\\n\\n\",\n        // Horizontal lines\n        \"\\n\\n***\\n\\n\",\n        \"\\n\\n---\\n\\n\",\n        \"\\n\\n___\\n\\n\",\n        // Note that this splitter doesn't handle horizontal lines defined\n        // by *three or more* of ***, ---, or ___, but this is not handled\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"latex\") {\n      return [\n        // First, try to split along Latex sections\n        \"\\n\\\\chapter{\",\n        \"\\n\\\\section{\",\n        \"\\n\\\\subsection{\",\n        \"\\n\\\\subsubsection{\",\n\n        // Now split by environments\n        \"\\n\\\\begin{enumerate}\",\n        \"\\n\\\\begin{itemize}\",\n        \"\\n\\\\begin{description}\",\n        \"\\n\\\\begin{list}\",\n        \"\\n\\\\begin{quote}\",\n        \"\\n\\\\begin{quotation}\",\n        \"\\n\\\\begin{verse}\",\n        \"\\n\\\\begin{verbatim}\",\n\n        // Now split by math environments\n        \"\\n\\\\begin{align}\",\n        \"$$\",\n        \"$\",\n\n        // Now split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"html\") {\n      return [\n        // First, try to split along HTML tags\n        \"<body>\",\n        \"<div>\",\n        \"<p>\",\n        \"<br>\",\n        \"<li>\",\n        \"<h1>\",\n        \"<h2>\",\n        \"<h3>\",\n        \"<h4>\",\n        \"<h5>\",\n        \"<h6>\",\n        \"<span>\",\n        \"<table>\",\n        \"<tr>\",\n        \"<td>\",\n        \"<th>\",\n        \"<ul>\",\n        \"<ol>\",\n        \"<header>\",\n        \"<footer>\",\n        \"<nav>\",\n        // Head\n        \"<head>\",\n        \"<style>\",\n        \"<script>\",\n        \"<meta>\",\n        \"<title>\",\n        // Normal type of lines\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"sol\") {\n      return [\n        // Split along compiler informations definitions\n        \"\\npragma \",\n        \"\\nusing \",\n        // Split along contract definitions\n        \"\\ncontract \",\n        \"\\ninterface \",\n        \"\\nlibrary \",\n        // Split along method definitions\n        \"\\nconstructor \",\n        \"\\ntype \",\n        \"\\nfunction \",\n        \"\\nevent \",\n        \"\\nmodifier \",\n        \"\\nerror \",\n        \"\\nstruct \",\n        \"\\nenum \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\ndo while \",\n        \"\\nassembly \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else {\n      throw new Error(`Language ${language} is not supported.`);\n    }\n  }\n}\n\nexport interface TokenTextSplitterParams extends TextSplitterParams {\n  encodingName: tiktoken.TiktokenEncoding;\n  allowedSpecial: \"all\" | Array<string>;\n  disallowedSpecial: \"all\" | Array<string>;\n}\n\n/**\n * Implementation of splitter which looks at tokens.\n */\nexport class TokenTextSplitter\n  extends TextSplitter\n  implements TokenTextSplitterParams\n{\n  static lc_name() {\n    return \"TokenTextSplitter\";\n  }\n\n  encodingName: tiktoken.TiktokenEncoding;\n\n  allowedSpecial: \"all\" | Array<string>;\n\n  disallowedSpecial: \"all\" | Array<string>;\n\n  private tokenizer: tiktoken.Tiktoken;\n\n  constructor(fields?: Partial<TokenTextSplitterParams>) {\n    super(fields);\n\n    this.encodingName = fields?.encodingName ?? \"gpt2\";\n    this.allowedSpecial = fields?.allowedSpecial ?? [];\n    this.disallowedSpecial = fields?.disallowedSpecial ?? \"all\";\n  }\n\n  async splitText(text: string): Promise<string[]> {\n    if (!this.tokenizer) {\n      this.tokenizer = await getEncoding(this.encodingName);\n    }\n\n    const splits: string[] = [];\n\n    const input_ids = this.tokenizer.encode(\n      text,\n      this.allowedSpecial,\n      this.disallowedSpecial\n    );\n\n    let start_idx = 0;\n\n    while (start_idx < input_ids.length) {\n      if (start_idx > 0) {\n        start_idx -= this.chunkOverlap;\n      }\n      const end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);\n      const chunk_ids = input_ids.slice(start_idx, end_idx);\n      splits.push(this.tokenizer.decode(chunk_ids));\n      start_idx = end_idx;\n    }\n\n    return splits;\n  }\n}\n\nexport type MarkdownTextSplitterParams = TextSplitterParams;\n\nexport class MarkdownTextSplitter\n  extends RecursiveCharacterTextSplitter\n  implements MarkdownTextSplitterParams\n{\n  constructor(fields?: Partial<MarkdownTextSplitterParams>) {\n    super({\n      ...fields,\n      separators:\n        RecursiveCharacterTextSplitter.getSeparatorsForLanguage(\"markdown\"),\n    });\n  }\n}\n\nexport type LatexTextSplitterParams = TextSplitterParams;\n\nexport class LatexTextSplitter\n  extends RecursiveCharacterTextSplitter\n  implements LatexTextSplitterParams\n{\n  constructor(fields?: Partial<LatexTextSplitterParams>) {\n    super({\n      ...fields,\n      separators:\n        RecursiveCharacterTextSplitter.getSeparatorsForLanguage(\"latex\"),\n    });\n  }\n}\n"],"mappings":";;;;;AAmBA,IAAsB,eAAtB,cACUA,mDAEV;CACE,eAAe;EAAC;EAAa;EAAyB;CAAiB;CAEvE,YAAY;CAEZ,eAAe;CAEf,gBAAgB;CAEhB;CAIA,YAAYC,QAAsC;EAChD,MAAM,OAAO;EACb,KAAK,YAAY,QAAQ,aAAa,KAAK;EAC3C,KAAK,eAAe,QAAQ,gBAAgB,KAAK;EACjD,KAAK,gBAAgB,QAAQ,iBAAiB,KAAK;EACnD,KAAK,iBACH,QAAQ,mBAAmB,CAACC,SAAiB,KAAK;AACpD,MAAI,KAAK,gBAAgB,KAAK,UAC5B,OAAM,IAAI,MAAM;CAEnB;CAED,MAAM,mBACJC,WACAC,qBAAqD,CAAE,GAClC;AACrB,SAAO,KAAK,eAAe,WAAW,mBAAmB;CAC1D;CAID,AAAU,iBAAiBF,MAAcG,WAA6B;EACpE,IAAI;AACJ,MAAI,UACF,KAAI,KAAK,eAAe;GACtB,MAAM,wBAAwB,UAAU,QACtC,0BACA,OACD;GACD,SAAS,KAAK,sBAAM,IAAI,OAAO,CAAC,GAAG,EAAE,sBAAsB,CAAC,CAAC,EAAE;EAChE,OACC,SAAS,KAAK,MAAM,UAAU;OAGhC,SAAS,KAAK,MAAM,GAAG;AAEzB,SAAO,OAAO,OAAO,CAAC,MAAM,MAAM,GAAG;CACtC;CAED,MAAM,gBACJC,OAEAC,YAAmC,CAAE,GACrCH,qBAAqD,CAAE,GAClC;EAGrB,MAAMI,aACJ,UAAU,SAAS,IACf,YACA,CAAC,GAAG,MAAM,MAAM,OAAO,AAAC,EAAC,IAAI,OAAO,CAAE,GAAE;EAC9C,MAAM,EACJ,cAAc,IACd,qBAAqB,aACrB,2BAA2B,OAC5B,GAAG;EACJ,MAAM,YAAY,IAAI;AACtB,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,OAAO,MAAM;GACnB,IAAI,mBAAmB;GACvB,IAAI,YAAY;GAChB,IAAI,iBAAiB;AACrB,QAAK,MAAM,SAAS,MAAM,KAAK,UAAU,KAAK,EAAE;IAC9C,IAAI,cAAc;IAGlB,MAAM,aAAa,KAAK,QAAQ,OAAO,iBAAiB,EAAE;AAC1D,QAAI,cAAc,MAAM;KACtB,MAAM,2BAA2B,KAAK,iBACpC,MACA,GACA,WACD;KACD,oBAAoB;IACrB,OAAM;KACL,MAAM,oBACJ,iBAAkB,MAAM,KAAK,eAAe,UAAU;AACxD,SAAI,oBAAoB,YAAY;MAClC,MAAM,+BAA+B,KAAK,iBACxC,MACA,mBACA,WACD;MACD,oBAAoB;KACrB,WAAU,oBAAoB,YAAY;MACzC,MAAM,+BAA+B,KAAK,iBACxC,MACA,YACA,kBACD;MACD,oBAAoB;KACrB;AACD,SAAI,0BACF,eAAe;IAElB;IACD,MAAM,gBAAgB,KAAK,iBAAiB,MAAM;IAElD,MAAM,MACJ,WAAW,GAAG,OAAO,OAAO,WAAW,GAAG,QAAQ,WAC9C,EAAE,GAAG,WAAW,GAAG,IAAK,IACxB,CAAE;IACR,IAAI,QAAQ;KACV,MAAM;KACN,IAAI,mBAAmB;IACxB;IACD,MAAM,0BAA0B;KAC9B,GAAG,WAAW;KACd;IACD;IAED,eAAe;IACf,UAAU,KACR,IAAIC,oCAAS;KACX;KACA,UAAU;IACX,GACF;IACD,oBAAoB;IACpB,YAAY;IACZ,iBAAiB;GAClB;EACF;AACD,SAAO;CACR;CAED,AAAQ,iBAAiBP,MAAcQ,OAAgBC,KAAc;EACnE,MAAM,cAAc,KAAK,MAAM,OAAO,IAAI;AAC1C,UAAQ,YAAY,MAAM,MAAM,IAAI,CAAE,GAAE;CACzC;CAED,MAAM,eACJR,WACAC,qBAAqD,CAAE,GAClC;EACrB,MAAM,oBAAoB,UAAU,OAClC,CAAC,QAAQ,IAAI,gBAAgB,OAC9B;EACD,MAAM,QAAQ,kBAAkB,IAAI,CAAC,QAAQ,IAAI,YAAY;EAC7D,MAAM,YAAY,kBAAkB,IAAI,CAAC,QAAQ,IAAI,SAAS;AAC9D,SAAO,KAAK,gBAAgB,OAAO,WAAW,mBAAmB;CAClE;CAED,AAAQ,SAASQ,MAAgBP,WAAkC;EACjE,MAAM,OAAO,KAAK,KAAK,UAAU,CAAC,MAAM;AACxC,SAAO,SAAS,KAAK,OAAO;CAC7B;CAED,MAAM,YAAYQ,QAAkBR,WAAsC;EACxE,MAAMO,OAAiB,CAAE;EACzB,MAAME,aAAuB,CAAE;EAC/B,IAAI,QAAQ;AACZ,OAAK,MAAM,KAAK,QAAQ;GACtB,MAAM,OAAO,MAAM,KAAK,eAAe,EAAE;AACzC,OACE,QAAQ,OAAO,WAAW,SAAS,UAAU,SAC7C,KAAK,WACL;AACA,QAAI,QAAQ,KAAK,WACf,QAAQ,KACN,CAAC,wBAAwB,EAAE,MAAM;mCACV,EAAE,KAAK,WAAW,CAC1C;AAEH,QAAI,WAAW,SAAS,GAAG;KACzB,MAAMC,QAAM,KAAK,SAAS,YAAY,UAAU;AAChD,SAAIA,UAAQ,MACV,KAAK,KAAKA,MAAI;AAKhB,YACE,QAAQ,KAAK,gBACZ,QAAQ,OAAO,WAAW,SAAS,UAAU,SAC5C,KAAK,aACL,QAAQ,GACV;MACA,SAAS,MAAM,KAAK,eAAe,WAAW,GAAG;MACjD,WAAW,OAAO;KACnB;IACF;GACF;GACD,WAAW,KAAK,EAAE;GAClB,SAAS;EACV;EACD,MAAM,MAAM,KAAK,SAAS,YAAY,UAAU;AAChD,MAAI,QAAQ,MACV,KAAK,KAAK,IAAI;AAEhB,SAAO;CACR;AACF;AAMD,IAAa,wBAAb,cACU,aAEV;CACE,OAAO,UAAU;AACf,SAAO;CACR;CAED,YAAY;CAEZ,YAAYC,QAA+C;EACzD,MAAM,OAAO;EACb,KAAK,YAAY,QAAQ,aAAa,KAAK;CAC5C;CAED,MAAM,UAAUd,MAAiC;EAE/C,MAAM,SAAS,KAAK,iBAAiB,MAAM,KAAK,UAAU;AAC1D,SAAO,KAAK,YAAY,QAAQ,KAAK,gBAAgB,KAAK,KAAK,UAAU;CAC1E;AACF;AAOD,MAAa,iCAAiC;CAC5C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACD;AAKD,IAAa,iCAAb,MAAa,uCACH,aAEV;CACE,OAAO,UAAU;AACf,SAAO;CACR;CAED,aAAuB;EAAC;EAAQ;EAAM;EAAK;CAAG;CAE9C,YAAYe,QAAwD;EAClE,MAAM,OAAO;EACb,KAAK,aAAa,QAAQ,cAAc,KAAK;EAC7C,KAAK,gBAAgB,QAAQ,iBAAiB;CAC/C;CAED,MAAc,WAAWf,MAAcgB,YAAsB;EAC3D,MAAMC,cAAwB,CAAE;EAGhC,IAAId,YAAoB,WAAW,WAAW,SAAS;EACvD,IAAI;AACJ,OAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK,GAAG;GAC7C,MAAM,IAAI,WAAW;AACrB,OAAI,MAAM,IAAI;IACZ,YAAY;AACZ;GACD;AACD,OAAI,KAAK,SAAS,EAAE,EAAE;IACpB,YAAY;IACZ,gBAAgB,WAAW,MAAM,IAAI,EAAE;AACvC;GACD;EACF;EAGD,MAAM,SAAS,KAAK,iBAAiB,MAAM,UAAU;EAGrD,IAAIe,aAAuB,CAAE;EAC7B,MAAM,aAAa,KAAK,gBAAgB,KAAK;AAC7C,OAAK,MAAM,KAAK,OACd,KAAK,MAAM,KAAK,eAAe,EAAE,GAAI,KAAK,WACxC,WAAW,KAAK,EAAE;OACb;AACL,OAAI,WAAW,QAAQ;IACrB,MAAM,aAAa,MAAM,KAAK,YAAY,YAAY,WAAW;IACjE,YAAY,KAAK,GAAG,WAAW;IAC/B,aAAa,CAAE;GAChB;AACD,OAAI,CAAC,eACH,YAAY,KAAK,EAAE;QACd;IACL,MAAM,YAAY,MAAM,KAAK,WAAW,GAAG,cAAc;IACzD,YAAY,KAAK,GAAG,UAAU;GAC/B;EACF;AAEH,MAAI,WAAW,QAAQ;GACrB,MAAM,aAAa,MAAM,KAAK,YAAY,YAAY,WAAW;GACjE,YAAY,KAAK,GAAG,WAAW;EAChC;AACD,SAAO;CACR;CAED,MAAM,UAAUlB,MAAiC;AAC/C,SAAO,KAAK,WAAW,MAAM,KAAK,WAAW;CAC9C;CAED,OAAO,aACLmB,UACAC,SACA;AACA,SAAO,IAAI,+BAA+B;GACxC,GAAG;GACH,YACE,+BAA+B,yBAAyB,SAAS;EACpE;CACF;CAED,OAAO,yBAAyBD,UAAyC;AACvE,MAAI,aAAa,MACf,QAAO;GAEL;GAEA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,KACtB,QAAO;GAEL;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,OACtB,QAAO;GAEL;GAEA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,KACtB,QAAO;GAEL;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,MACtB,QAAO;GAEL;GAEA;GAEA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,QACtB,QAAO;GAEL;GAEA;GAEA;GAEA;GAEA;GAEA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,SACtB,QAAO;GAEL;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,MACtB,QAAO;GAEL;GACA;GACA;GAEA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,OACtB,QAAO;GAEL;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,OACtB,QAAO;GAEL;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,QACtB,QAAO;GAEL;GACA;GAEA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,QACtB,QAAO;GAEL;GAEA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;WACQ,aAAa,WACtB,QAAO;GAEL;GACA;GACA;GACA;GACA;GAKA;GAEA;GACA;GACA;GAGA;GACA;GACA;GACA;EACD;WACQ,aAAa,QACtB,QAAO;GAEL;GACA;GACA;GACA;GAGA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GAGA;GACA;GACA;GAGA;GACA;GACA;GACA;EACD;WACQ,aAAa,OACtB,QAAO;GAEL;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;EACD;WACQ,aAAa,MACtB,QAAO;GAEL;GACA;GAEA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;EACD;MAED,OAAM,IAAI,MAAM,CAAC,SAAS,EAAE,SAAS,kBAAkB,CAAC;CAE3D;AACF;;;;AAWD,IAAa,oBAAb,cACU,aAEV;CACE,OAAO,UAAU;AACf,SAAO;CACR;CAED;CAEA;CAEA;CAEA,AAAQ;CAER,YAAYE,QAA2C;EACrD,MAAM,OAAO;EAEb,KAAK,eAAe,QAAQ,gBAAgB;EAC5C,KAAK,iBAAiB,QAAQ,kBAAkB,CAAE;EAClD,KAAK,oBAAoB,QAAQ,qBAAqB;CACvD;CAED,MAAM,UAAUrB,MAAiC;AAC/C,MAAI,CAAC,KAAK,WACR,KAAK,YAAY,uDAAkB,KAAK,aAAa;EAGvD,MAAMW,SAAmB,CAAE;EAE3B,MAAM,YAAY,KAAK,UAAU,OAC/B,MACA,KAAK,gBACL,KAAK,kBACN;EAED,IAAI,YAAY;AAEhB,SAAO,YAAY,UAAU,QAAQ;AACnC,OAAI,YAAY,GACd,aAAa,KAAK;GAEpB,MAAM,UAAU,KAAK,IAAI,YAAY,KAAK,WAAW,UAAU,OAAO;GACtE,MAAM,YAAY,UAAU,MAAM,WAAW,QAAQ;GACrD,OAAO,KAAK,KAAK,UAAU,OAAO,UAAU,CAAC;GAC7C,YAAY;EACb;AAED,SAAO;CACR;AACF;AAID,IAAa,uBAAb,cACU,+BAEV;CACE,YAAYW,QAA8C;EACxD,MAAM;GACJ,GAAG;GACH,YACE,+BAA+B,yBAAyB,WAAW;EACtE,EAAC;CACH;AACF;AAID,IAAa,oBAAb,cACU,+BAEV;CACE,YAAYC,QAA2C;EACrD,MAAM;GACJ,GAAG;GACH,YACE,+BAA+B,yBAAyB,QAAQ;EACnE,EAAC;CACH;AACF"}