{"version":3,"file":"chunking.mjs","names":[],"sources":["../src/chunking.ts"],"sourcesContent":["import { Context, Effect, Layer } from 'effect'\nimport { countTokens, decode, encode } from 'gpt-tokenizer/encoding/o200k_base'\nimport type {\n  KnowledgeChunk,\n  KnowledgeDocumentId,\n  KnowledgeMetadata,\n  KnowledgeScopeId\n} from './documents.ts'\nimport { KnowledgeChunkingError } from './errors.ts'\n\nconst SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:[\"')\\]]+)?\\s*|[^.!?]+$/g\n\nconst PARAGRAPH_BREAK_PATTERN = /(\\n{2,})/\n\nconst WORD_PATTERN = /\\S+\\s*/g\n\nexport type ChunkKnowledgeDocumentInput = {\n  readonly scopeId: KnowledgeScopeId\n  readonly documentId: KnowledgeDocumentId\n  readonly content: string\n  readonly maxTokens?: number\n  readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeChunkerApi = {\n  readonly chunk: (\n    input: ChunkKnowledgeDocumentInput\n  ) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, KnowledgeChunkingError>\n}\n\nexport class KnowledgeChunker extends Context.Service<KnowledgeChunker, KnowledgeChunkerApi>()(\n  '@yolk-sdk/knowledge/KnowledgeChunker'\n) {}\n\nexport const countKnowledgeChunkTokens = (text: string) => countTokens(text)\n\nconst sanitizeText = (text: string) =>\n  text\n    .replace(/\\r\\n/g, '\\n')\n    .replace(/[\\t ]+/g, ' ')\n    .replace(/\\n{3,}/g, '\\n\\n')\n    .trim()\n\nconst splitSentences = (text: string) =>\n  text\n    .split(PARAGRAPH_BREAK_PATTERN)\n    .flatMap(part => {\n      if (part.length === 0) {\n        return []\n      }\n\n      if (/^\\n{2,}$/.test(part)) {\n        return ['\\n\\n']\n      }\n\n      return part.match(SENTENCE_PATTERN) ?? [part]\n    })\n    .filter(unit => unit.length > 0)\n\nconst splitEncodedTokens = (text: string, maxTokens: number) => {\n  const tokens = encode(text)\n\n  return Array.from({ length: Math.ceil(tokens.length / maxTokens) }, (_, index) =>\n    decode(tokens.slice(index * maxTokens, index * maxTokens + maxTokens)).trim()\n  ).filter(chunk => chunk.length > 0)\n}\n\nconst splitOversizedUnit = (text: string, maxTokens: number) => {\n  const words = text.match(WORD_PATTERN) ?? []\n  const chunks: string[] = []\n  let current = ''\n  let currentTokenCount = 0\n\n  for (const word of words) {\n    const wordTokenCount = countTokens(word)\n\n    if (wordTokenCount > maxTokens) {\n      const currentChunk = current.trim()\n\n      if (currentChunk.length > 0) {\n        chunks.push(currentChunk)\n      }\n\n      chunks.push(...splitEncodedTokens(word, maxTokens))\n      current = ''\n      currentTokenCount = 0\n      continue\n    }\n\n    if (currentTokenCount > 0 && currentTokenCount + wordTokenCount > maxTokens) {\n      const currentChunk = current.trim()\n\n      if (currentChunk.length > 0) {\n        chunks.push(currentChunk)\n      }\n\n      current = word\n      currentTokenCount = wordTokenCount\n      continue\n    }\n\n    current = `${current}${word}`\n    currentTokenCount += wordTokenCount\n  }\n\n  const finalChunk = current.trim()\n\n  if (finalChunk.length > 0) {\n    chunks.push(finalChunk)\n  }\n\n  return chunks\n}\n\nconst splitUnitsBySize = (units: ReadonlyArray<string>, maxTokens: number) =>\n  units.flatMap(unit =>\n    countTokens(unit) <= maxTokens ? [unit] : splitOversizedUnit(unit, maxTokens)\n  )\n\nconst buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) => {\n  const chunks: string[] = []\n  let current = ''\n  let currentTokenCount = 0\n\n  const pushCurrent = () => {\n    const content = current.trim()\n\n    if (content.length > 0) {\n      chunks.push(content)\n    }\n  }\n\n  for (const unit of units) {\n    const unitTokenCount = countTokens(unit)\n\n    if (currentTokenCount > 0 && currentTokenCount + unitTokenCount > maxTokens) {\n      pushCurrent()\n      current = unit\n      currentTokenCount = unitTokenCount\n      continue\n    }\n\n    current = `${current}${unit}`\n    currentTokenCount += unitTokenCount\n  }\n\n  pushCurrent()\n\n  return chunks\n}\n\nexport const chunkKnowledgeText = (input: ChunkKnowledgeDocumentInput, maxTokens: number) =>\n  Effect.gen(function* () {\n    if (!Number.isInteger(maxTokens) || maxTokens < 1) {\n      return yield* Effect.fail(\n        new KnowledgeChunkingError({ message: 'Chunk maxTokens must be a positive integer' })\n      )\n    }\n\n    const sanitized = sanitizeText(input.content)\n\n    if (sanitized.length === 0) {\n      return yield* Effect.fail(\n        new KnowledgeChunkingError({ message: 'Cannot chunk empty content' })\n      )\n    }\n\n    const units = splitUnitsBySize(splitSentences(sanitized), maxTokens)\n    const contents = buildChunkContents(units, maxTokens)\n\n    if (contents.length === 0) {\n      return yield* Effect.fail(new KnowledgeChunkingError({ message: 'No chunks produced' }))\n    }\n\n    return contents.map((content, position) => ({\n      id: `${input.documentId}:chunk:${position}`,\n      scopeId: input.scopeId,\n      documentId: input.documentId,\n      content,\n      position,\n      tokenCount: countTokens(content),\n      metadata: input.metadata\n    })) satisfies ReadonlyArray<KnowledgeChunk>\n  })\n\nexport const makeDefaultKnowledgeChunker = (input: {\n  readonly maxTokens: number\n}): KnowledgeChunkerApi => ({\n  chunk: document => chunkKnowledgeText(document, document.maxTokens ?? input.maxTokens)\n})\n\nexport const DefaultKnowledgeChunkerLive = (\n  input: { readonly maxTokens: number } = { maxTokens: 512 }\n) => Layer.succeed(KnowledgeChunker, makeDefaultKnowledgeChunker(input))\n"],"mappings":";;;;AAUA,MAAM,mBAAmB;AAEzB,MAAM,0BAA0B;AAEhC,MAAM,eAAe;AAgBrB,IAAa,mBAAb,cAAsC,QAAQ,QAA+C,EAC3F,sCACF,EAAE,CAAC;AAEH,MAAa,6BAA6B,SAAiB,YAAY,IAAI;AAE3E,MAAM,gBAAgB,SACpB,KACG,QAAQ,SAAS,IAAI,EACrB,QAAQ,WAAW,GAAG,EACtB,QAAQ,WAAW,MAAM,EACzB,KAAK;AAEV,MAAM,kBAAkB,SACtB,KACG,MAAM,uBAAuB,EAC7B,SAAQ,SAAQ;CACf,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAGV,IAAI,WAAW,KAAK,IAAI,GACtB,OAAO,CAAC,MAAM;CAGhB,OAAO,KAAK,MAAM,gBAAgB,KAAK,CAAC,IAAI;AAC9C,CAAC,EACA,QAAO,SAAQ,KAAK,SAAS,CAAC;AAEnC,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,SAAS,OAAO,IAAI;CAE1B,OAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,KAAK,OAAO,SAAS,SAAS,EAAE,IAAI,GAAG,UACtE,OAAO,OAAO,MAAM,QAAQ,WAAW,QAAQ,YAAY,SAAS,CAAC,EAAE,KAAK,CAC9E,EAAE,QAAO,UAAS,MAAM,SAAS,CAAC;AACpC;AAEA,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,QAAQ,KAAK,MAAM,YAAY,KAAK,CAAC;CAC3C,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,iBAAiB,WAAW;GAC9B,MAAM,eAAe,QAAQ,KAAK;GAElC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,OAAO,KAAK,GAAG,mBAAmB,MAAM,SAAS,CAAC;GAClD,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,MAAM,eAAe,QAAQ,KAAK;GAElC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,MAAM,aAAa,QAAQ,KAAK;CAEhC,IAAI,WAAW,SAAS,GACtB,OAAO,KAAK,UAAU;CAGxB,OAAO;AACT;AAEA,MAAM,oBAAoB,OAA8B,cACtD,MAAM,SAAQ,SACZ,YAAY,IAAI,KAAK,YAAY,CAAC,IAAI,IAAI,mBAAmB,MAAM,SAAS,CAC9E;AAEF,MAAM,sBAAsB,OAA8B,cAAsB;CAC9E,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,MAAM,oBAAoB;EACxB,MAAM,UAAU,QAAQ,KAAK;EAE7B,IAAI,QAAQ,SAAS,GACnB,OAAO,KAAK,OAAO;CAEvB;CAEA,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,YAAY;GACZ,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,YAAY;CAEZ,OAAO;AACT;AAEA,MAAa,sBAAsB,OAAoC,cACrE,OAAO,IAAI,aAAa;CACtB,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,OAAO,OAAO,OAAO,KACnB,IAAI,uBAAuB,EAAE,SAAS,6CAA6C,CAAC,CACtF;CAGF,MAAM,YAAY,aAAa,MAAM,OAAO;CAE5C,IAAI,UAAU,WAAW,GACvB,OAAO,OAAO,OAAO,KACnB,IAAI,uBAAuB,EAAE,SAAS,6BAA6B,CAAC,CACtE;CAIF,MAAM,WAAW,mBADH,iBAAiB,eAAe,SAAS,GAAG,SAClB,GAAG,SAAS;CAEpD,IAAI,SAAS,WAAW,GACtB,OAAO,OAAO,OAAO,KAAK,IAAI,uBAAuB,EAAE,SAAS,qBAAqB,CAAC,CAAC;CAGzF,OAAO,SAAS,KAAK,SAAS,cAAc;EAC1C,IAAI,GAAG,MAAM,WAAW,SAAS;EACjC,SAAS,MAAM;EACf,YAAY,MAAM;EAClB;EACA;EACA,YAAY,YAAY,OAAO;EAC/B,UAAU,MAAM;CAClB,EAAE;AACJ,CAAC;AAEH,MAAa,+BAA+B,WAEhB,EAC1B,QAAO,aAAY,mBAAmB,UAAU,SAAS,aAAa,MAAM,SAAS,EACvF;AAEA,MAAa,+BACX,QAAwC,EAAE,WAAW,IAAI,MACtD,MAAM,QAAQ,kBAAkB,4BAA4B,KAAK,CAAC"}