{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../../src/core/embsearch/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,eAAO,MAAM,eAAe,IAAI,CAAC;AAgBjC;;;;;;;;;GASG;AACH,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,MAAM,WAAW,KAAK;IACrB,6DAA2D;IAC3D,EAAE,EAAE,MAAM,CAAC;IACX,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,OAAO,EAAE,MAAM,CAAC;CAChB;AAQD;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAwB,GAAG,KAAK,EAAE,CA8CvG","sourcesContent":["/**\n * File chunking for semantic indexing.\n *\n * Splits a file into overlapping line windows, each capped by characters so a\n * chunk stays within the embedding model's effective token window (MiniLM\n * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`;\n * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts)\n * so search hits can be rendered as `path:start-end`.\n *\n * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the\n * sidecar triggers a clean rebuild of the store.\n */\n\nexport const CHUNKER_VERSION = 2;\n\n/**\n * Target lines per chunk.\n *\n * Halving this (30 lines / 500 chars) was measured and rejected. The theory\n * was dilution — a four-line answer sharing one vector with five neighbouring\n * methods — but sharper chunks cost reach: Recall@50 fell 79% -> 69% on\n * `auto +rr` and the boundary class went from 2 of 4 findable to 0 of 4.\n * A fixed top-k over smaller chunks retrieves less *content*: 50 chunks at\n * ~460 chars sees half the corpus that 50 at ~930 does. Recall@1 ticked up\n * (53% -> 55%), which is the sharpening, and it did not pay for the loss.\n */\nconst CHUNK_LINES = 60;\n/** Overlapping lines between consecutive chunks, for context continuity. */\nconst CHUNK_OVERLAP_LINES = 10;\n/**\n * Hard character cap per chunk.\n *\n * The \"~256 tokens ≈ 1000 chars\" this was set from does not hold for code:\n * measured against the bundled tokenizer over this repo, 1000 chars is **313\n * tokens** at the median, so MiniLM (256) truncates 85.8% of chunks and drops\n * 20.3% of the corpus's tokens, while bge-small (512) drops 0.1%. Raising this\n * is therefore not free in the way the old comment implied — it spends a\n * budget that is already overdrawn on one model and nearly full on the other.\n */\nexport const CHUNK_MAX_CHARS = 1000;\n\nexport interface Chunk {\n\t/** `relpath#index` — the id stored in the vector index. */\n\tid: string;\n\t/** Text sent to the embedder. */\n\ttext: string;\n\t/** 1-based inclusive start line. */\n\tstartLine: number;\n\t/** 1-based inclusive end line. */\n\tendLine: number;\n}\n\n/** Heuristic binary sniff: NUL byte in the first 8KB. */\nfunction looksBinary(content: string): boolean {\n\tconst probe = content.slice(0, 8192);\n\treturn probe.includes(\"\\u0000\");\n}\n\n/**\n * Split `content` into chunks. `relPath` becomes the id prefix. Returns an\n * empty array for empty or binary-looking content.\n *\n * `maxChars` overrides {@link CHUNK_MAX_CHARS} for eval arms that sweep the\n * window. Production never passes it; a caller that does is changing what the\n * index contains and owes the store a distinct key, since nothing about a\n * stored vector records the cap it was built under.\n */\nexport function chunkFile(relPath: string, content: string, maxChars: number = CHUNK_MAX_CHARS): Chunk[] {\n\tif (!content.trim() || looksBinary(content)) return [];\n\tconst lines = content.replace(/\\r\\n/g, \"\\n\").replace(/\\r/g, \"\\n\").split(\"\\n\");\n\tconst chunks: Chunk[] = [];\n\tlet start = 0; // 0-based\n\tlet index = 0;\n\n\twhile (start < lines.length) {\n\t\tlet end = start; // exclusive\n\t\tlet chars = 0;\n\t\twhile (end < lines.length && end - start < CHUNK_LINES) {\n\t\t\tconst lineLen = lines[end].length + 1;\n\t\t\tif (chars + lineLen > maxChars && end > start) break;\n\t\t\tchars += lineLen;\n\t\t\tend++;\n\t\t}\n\t\t// Trim whole blank lines off both ends, and narrow the recorded range to\n\t\t// match. Trimming the joined string instead left `startLine`/`endLine`\n\t\t// claiming lines whose text was never embedded — 31% of chunks in this\n\t\t// repo, up to 3 lines each. Those lines were unfindable by the vector\n\t\t// leg while still being handed to the context assembler as if they were\n\t\t// part of the chunk.\n\t\tlet from = start;\n\t\tlet to = end; // exclusive\n\t\twhile (from < to && lines[from].trim() === \"\") from++;\n\t\twhile (to > from && lines[to - 1].trim() === \"\") to--;\n\t\tlet text = lines.slice(from, to).join(\"\\n\").trim();\n\t\tif (text.length > maxChars) {\n\t\t\t// Oversized chunk (e.g. long minified line): keep the prefix. The\n\t\t\t// underlying model would truncate anyway, so this stays bounded.\n\t\t\ttext = text.slice(0, maxChars);\n\t\t}\n\t\tif (text) {\n\t\t\tchunks.push({\n\t\t\t\tid: `${relPath}#${index}`,\n\t\t\t\ttext,\n\t\t\t\tstartLine: from + 1,\n\t\t\t\tendLine: to,\n\t\t\t});\n\t\t\tindex++;\n\t\t}\n\t\tif (end >= lines.length) break;\n\t\t// Step forward with overlap, but always make progress.\n\t\tstart = Math.max(end - CHUNK_OVERLAP_LINES, start + 1);\n\t}\n\treturn chunks;\n}\n"]}