/** * Shared machinery for the splitter family. * * Pattern: internal helpers, not exported from the door. * Role: rag/ layer. The offset arithmetic lives here ONCE, because it is the * part every splitter must get exactly right and the part that is * easy to get subtly wrong per-strategy. * Emits: N/A. * * **The invariant, stated once:** for every piece produced, * `doc.text.slice(charStart, charEnd) === piece.text`. Every helper here * derives its text BY SLICING rather than by concatenating what it read, which * is what makes the invariant true by construction rather than by care. */ import type { LoadedDocument, SplitPiece } from '../types.js'; /** A half-open span of the source text. */ export interface Span { readonly start: number; readonly end: number; readonly heading?: string; /** * Where this span's OWN content begins, before overlap was prepended. * * Overlap deliberately reaches backward into the previous chunk, which means * `start` can sit on the previous PAGE. Attributing the chunk to that page * would cite a page whose text is only the borrowed run-up — so page lookup * uses this instead, and it is set only when the two differ. */ readonly contentStart?: number; } /** * Which page an offset falls on, 1-based, or `undefined` when the document is * not paginated. * * Pages were joined with `'\n\n'` by the loader, so the boundaries are * reconstructible exactly. A chunk that straddles a page break is attributed * to the page it STARTS on — one number, the one you would turn to first. */ export declare function pageLocator(doc: LoadedDocument): (offset: number) => number | undefined; /** * Trim a span inward past leading/trailing whitespace, keeping the offsets * honest. * * The naive version — `slice().trim()` — produces text that no longer matches * its own offsets. This moves the offsets instead, so the invariant holds. * Returns `undefined` when the span is entirely whitespace. */ export declare function trimSpan(text: string, span: Span): Span | undefined; /** * Pack consecutive units (paragraphs, lines) into spans of at most `maxChars`, * then apply `overlapChars` of backward overlap. * * Units are never split by this function — a single unit longer than `maxChars` * becomes its own oversized span, and the caller decides whether to hard-cut * it. That separation matters: a 4,000-character paragraph is a real thing in * real documents, and silently slicing it mid-word is a different decision from * packing. */ export declare function packSpans(units: readonly Span[], maxChars: number): Span[]; /** * Extend each span backward by `overlapChars`, without crossing the previous * span's start. * * Backward rather than forward so a chunk always ENDS on the boundary its * strategy chose — a heading section that ends mid-sentence because of overlap * would be a section that lies about where it ends. */ export declare function applyOverlap(spans: readonly Span[], overlapChars: number): Span[]; /** * Fold a too-small span back into its predecessor. * * A fragment of a few characters — a stray heading, a signature line — embeds * to noise and then competes for a top-K slot with real passages. * * **It never folds across a heading boundary.** A merged chunk carries the * FIRST span's heading, so folding a short section into the one before it * would produce a chunk labelled `heading="Alpha"` whose text is Beta's — a * citation that names the wrong section, which is worse than a small chunk. * Two spans merge only when they are already the same section. */ export declare function foldRunts(spans: readonly Span[], minChars?: number): Span[]; /** * Resolve the effective `minChars` floor for a splitter (8.20.0). * * Unset → `min(DEFAULT_MIN_CHARS, maxChars / 4)`: 250 at the default target, * shrinking proportionally when the caller asked for small chunks, so a * defaulted floor can never swallow the chunk size the caller chose. * * An EXPLICIT floor is validated instead of clamped: a floor at or above the * target would merge every chunk into its neighbour, which is a configuration * contradiction, not a preference — refused by name rather than rounded away. */ export declare function resolveMinChars(minChars: number | undefined, maxChars: number): number; /** * Merge sub-floor spans FORWARD into their next neighbour (8.20.0). * * For heading-less span families (paragraphs, and `byHeading`'s no-headings * fallback). A span under the floor is never dropped and never shipped alone: * * - adjacent short spans accumulate; the moment the accumulation reaches the * floor it is emitted as its own span (two 150-char paragraphs make a fine * 300-char chunk — better than pushing both into a full-sized neighbour); * - an accumulation still under the floor joins the NEXT full span, so the * short text leads the merged chunk; * - at the end of the document there is no next, so a trailing accumulation * merges BACKWARD into the last emitted span; * - a document that is nothing but short spans stays one merged span — * there is no neighbour, and "never dropped" wins. * * Merged spans slice the SOURCE (start of the first constituent to end of the * last), so the offset invariant holds by construction, as everywhere else. */ export declare function mergeShortSpansForward(spans: readonly Span[], minChars: number): Span[]; /** Turn spans into pieces, slicing the text so the invariant holds by construction. */ export declare function toPieces(doc: LoadedDocument, spans: readonly Span[]): SplitPiece[]; /** Hard-cut a span that no packing could fit, at `maxChars` with overlap. */ export declare function hardCut(span: Span, maxChars: number, overlapChars: number): Span[]; /** Split into paragraph spans on blank lines, keeping offsets. */ export declare function paragraphSpans(text: string): Span[]; //# sourceMappingURL=shared.d.ts.map