{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../../src/core/workspace/chunk.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,WAAW;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,eAAe,GAAG,iBAAiB,CAAC;IACpF,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,eAAe,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,UAAU,CAAC;IAC/E,IAAI,EAAE,MAAM,CAAC;CACb;AAED,eAAO,MAAM,eAAe,MAAM,CAAC;AACnC,eAAO,MAAM,eAAe,OAAO,CAAC;AAWpC,wBAAgB,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,EAAE,CAmD1D","sourcesContent":["/**\n * Deterministic chunking. Prefers symbol boundaries, then markdown/config\n * sections, then bounded line windows. Chunk identity is content-addressed.\n */\n\nimport { sha256 } from \"./guard.js\";\nimport type { ExtractedSymbol } from \"./symbols.js\";\n\nexport interface ChunkInput {\n\tfileId: string;\n\tcontent: string;\n\tcontentSha256: string;\n\tlanguageId?: string;\n\tclassification: string;\n\tsymbols?: ExtractedSymbol[];\n}\n\nexport interface ChunkOutput {\n\tchunkId: string;\n\tfileId: string;\n\tcontentSha256: string;\n\tstartLine: number;\n\tendLine: number;\n\tstartByte?: number;\n\tendByte?: number;\n\tlanguageId?: string;\n\tsymbolId?: string;\n\tchunkKind: \"symbol\" | \"section\" | \"paragraph\" | \"configuration\" | \"fallback_window\";\n\ttextHash: string;\n\tembeddingStatus: \"not_requested\" | \"pending\" | \"ready\" | \"failed\" | \"excluded\";\n\ttext: string;\n}\n\nexport const MAX_CHUNK_LINES = 160;\nexport const MAX_CHUNK_CHARS = 6000;\n\nfunction byteRange(content: string, startLine: number, endLine: number): [number, number] {\n\tconst lines = content.split(\"\\n\");\n\tlet start = 0;\n\tfor (let i = 0; i < startLine && i < lines.length; i++) start += lines[i].length + 1;\n\tlet end = start;\n\tfor (let i = startLine; i < endLine && i < lines.length; i++) end += lines[i].length + 1;\n\treturn [start, end];\n}\n\nexport function chunkFile(input: ChunkInput): ChunkOutput[] {\n\tconst content = input.content.replace(/^\\uFEFF/, \"\");\n\tconst normalized = content.replace(/\\r\\n/g, \"\\n\").replace(/\\r/g, \"\\n\");\n\tconst lines = normalized.split(\"\\n\");\n\tconst chunks: ChunkOutput[] = [];\n\tconst isConfig = [\"configuration\", \"schema\", \"lockfile\"].includes(input.classification);\n\n\tif (isConfig) {\n\t\tchunks.push(...chunkConfig(input, normalized, lines));\n\t\treturn chunks;\n\t}\n\n\tif (input.languageId === \"markdown\" || input.classification === \"documentation\") {\n\t\tchunks.push(...chunkMarkdown(input, normalized, lines));\n\t\treturn chunks;\n\t}\n\n\t// Symbol-bounded chunking.\n\tif (input.symbols && input.symbols.length > 0) {\n\t\tconst sorted = [...input.symbols].sort((a, b) => a.startLine - b.startLine);\n\t\tlet cursor = 0;\n\t\tconst made = new Set<number>();\n\t\tfor (const sym of sorted) {\n\t\t\tif (sym.startLine < cursor) continue;\n\t\t\tconst start = sym.startLine;\n\t\t\tconst end = Math.min(sym.endLine, Math.max(start + 1, lines.length));\n\t\t\tif (end > start) {\n\t\t\t\tconst text = lines.slice(start, end).join(\"\\n\");\n\t\t\t\tconst chunk = makeChunk(input, start, end, text, \"symbol\", sym);\n\t\t\t\tif (!made.has(chunk.startLine)) {\n\t\t\t\t\tchunks.push(chunk);\n\t\t\t\t\tmade.add(chunk.startLine);\n\t\t\t\t}\n\t\t\t}\n\t\t\t// import/header prologue before the first symbol\n\t\t\tif (cursor < start && chunks.length === 0) {\n\t\t\t\tconst head = lines.slice(cursor, Math.min(start, cursor + 20)).join(\"\\n\");\n\t\t\t\tif (head.trim())\n\t\t\t\t\tchunks.unshift(makeChunk(input, cursor, Math.min(start, cursor + 20), head, \"section\", undefined));\n\t\t\t}\n\t\t\tcursor = Math.max(cursor, end);\n\t\t}\n\t\tif (chunks.length === 0 && lines.length) {\n\t\t\tchunks.push(makeChunk(input, 0, lines.length, lines.join(\"\\n\"), \"fallback_window\", undefined));\n\t\t}\n\t\treturn chunks;\n\t}\n\n\t// Fallback bounded line windows.\n\tchunks.push(...fallbackWindows(input, normalized, lines));\n\treturn chunks;\n}\n\nfunction makeChunk(\n\tinput: ChunkInput,\n\tstart: number,\n\tend: number,\n\trawText: string,\n\tkind: ChunkOutput[\"chunkKind\"],\n\tsymbol?: ExtractedSymbol,\n): ChunkOutput {\n\tlet text = rawText;\n\tlet effEnd = end;\n\t// Split oversized symbol chunks deterministically.\n\tif (text.length > MAX_CHUNK_CHARS) {\n\t\tconst maxLines = Math.max(20, Math.floor((MAX_CHUNK_CHARS / Math.max(1, text.length)) * (end - start)));\n\t\tconst slice = text.split(\"\\n\").slice(0, maxLines).join(\"\\n\");\n\t\ttext = slice;\n\t\teffEnd = start + slice.split(\"\\n\").length;\n\t}\n\tconst [sb, eb] = byteRange(input.content, start, effEnd);\n\treturn {\n\t\tchunkId: sha256(`${input.fileId}:${start}:${effEnd}:${sha256(text).slice(0, 16)}`).slice(0, 32),\n\t\tfileId: input.fileId,\n\t\tcontentSha256: input.contentSha256,\n\t\tstartLine: start,\n\t\tendLine: effEnd,\n\t\tstartByte: sb,\n\t\tendByte: eb,\n\t\tlanguageId: input.languageId,\n\t\tsymbolId: symbol ? sha256(`${input.fileId}:${symbol.qualifiedName ?? symbol.name}`).slice(0, 32) : undefined,\n\t\tchunkKind: kind,\n\t\ttextHash: sha256(text),\n\t\tembeddingStatus: \"not_requested\",\n\t\ttext,\n\t};\n}\n\nfunction fallbackWindows(input: ChunkInput, normalized: string, lines: string[]): ChunkOutput[] {\n\tconst out: ChunkOutput[] = [];\n\tconst step = 60;\n\tfor (let i = 0; i < lines.length; i += step) {\n\t\tconst start = i;\n\t\tconst end = Math.min(i + step, lines.length);\n\t\tconst text = lines.slice(start, end).join(\"\\n\");\n\t\tout.push(makeChunk(input, start, end, text, \"fallback_window\", undefined));\n\t}\n\tif (out.length === 0 && lines.length) {\n\t\tout.push(makeChunk(input, 0, lines.length, normalized, \"fallback_window\", undefined));\n\t}\n\treturn out;\n}\n\nfunction chunkMarkdown(input: ChunkInput, normalized: string, lines: string[]): ChunkOutput[] {\n\tconst headers = lines.map((l, i) => ({ i, m: l.match(/^(#{1,6})\\s+(.+)/) })).filter((x) => x.m);\n\tconst out: ChunkOutput[] = [];\n\tif (headers.length === 0) return fallbackWindows(input, normalized, lines);\n\tfor (let h = 0; h < headers.length; h++) {\n\t\tconst start = headers[h].i;\n\t\tconst end = h + 1 < headers.length ? headers[h + 1].i : lines.length;\n\t\tconst text = lines.slice(start, end).join(\"\\n\");\n\t\tout.push(makeChunk(input, start, end, text, \"section\", undefined));\n\t}\n\treturn out;\n}\n\nfunction chunkConfig(input: ChunkInput, normalized: string, lines: string[]): ChunkOutput[] {\n\t// Split top-level keys for YAML/TOML/properties; for JSON, single chunk if small.\n\tconst out: ChunkOutput[] = [];\n\tif (input.languageId === \"json\" && normalized.length <= MAX_CHUNK_CHARS) {\n\t\tout.push(makeChunk(input, 0, lines.length, normalized, \"configuration\", undefined));\n\t\treturn out;\n\t}\n\tlet sectionStart = 0;\n\tfor (let i = 0; i < lines.length; i++) {\n\t\tconst line = lines[i];\n\t\tconst isTopLevel = /^[A-Za-z0-9_\".'`-]+\\s*[:=]/.test(line) && !line.startsWith(\" \") && !line.startsWith(\"\\t\");\n\t\tif (isTopLevel && i > 0) {\n\t\t\tconst text = lines.slice(sectionStart, i).join(\"\\n\");\n\t\t\tif (text.trim()) out.push(makeChunk(input, sectionStart, i, text, \"configuration\", undefined));\n\t\t\tsectionStart = i;\n\t\t}\n\t}\n\tif (sectionStart < lines.length) {\n\t\tconst text = lines.slice(sectionStart).join(\"\\n\");\n\t\tif (text.trim()) out.push(makeChunk(input, sectionStart, lines.length, text, \"configuration\", undefined));\n\t}\n\tif (out.length === 0) out.push(makeChunk(input, 0, lines.length, normalized, \"configuration\", undefined));\n\treturn out;\n}\n"]}