/** * File chunking for semantic indexing. * * Splits a file into overlapping line windows, each capped by characters so a * chunk stays within the embedding model's effective token window (MiniLM * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`; * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts) * so search hits can be rendered as `path:start-end`. * * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the * sidecar triggers a clean rebuild of the store. */ export declare const CHUNKER_VERSION = 2; /** * Hard character cap per chunk. * * The "~256 tokens ≈ 1000 chars" this was set from does not hold for code: * measured against the bundled tokenizer over this repo, 1000 chars is **313 * tokens** at the median, so MiniLM (256) truncates 85.8% of chunks and drops * 20.3% of the corpus's tokens, while bge-small (512) drops 0.1%. Raising this * is therefore not free in the way the old comment implied — it spends a * budget that is already overdrawn on one model and nearly full on the other. */ export declare const CHUNK_MAX_CHARS = 1000; export interface Chunk { /** `relpath#index` — the id stored in the vector index. */ id: string; /** Text sent to the embedder. */ text: string; /** 1-based inclusive start line. */ startLine: number; /** 1-based inclusive end line. */ endLine: number; } /** * Split `content` into chunks. `relPath` becomes the id prefix. Returns an * empty array for empty or binary-looking content. * * `maxChars` overrides {@link CHUNK_MAX_CHARS} for eval arms that sweep the * window. Production never passes it; a caller that does is changing what the * index contains and owes the store a distinct key, since nothing about a * stored vector records the cap it was built under. */ export declare function chunkFile(relPath: string, content: string, maxChars?: number): Chunk[]; //# sourceMappingURL=chunker.d.ts.map