import type { TypedScope } from 'footprintjs'; import type { Embedder } from '../memory/embedding/index.js'; import type { MemoryStore } from '../memory/store/index.js'; import type { MemoryIdentity } from '../memory/identity/index.js'; import type { Chunk, DocumentLoader, DocumentSource, FailedDocument, IndexReport, LoadedDocument, Splitter, TruncatedChunk } from './types.js'; export interface IndexCorpusConfig { /** Where the documents come from. */ readonly source: DocumentSource; /** Where the vectors go. Any `MemoryStore`; `sqliteVectorStore` to keep them. */ readonly store: MemoryStore; /** What turns chunk text into vectors. */ readonly embedder: Embedder; /** * The namespace to index into. Defaults to the same * `{ conversationId: '_global' }` that `defineRAG` reads from, so index with * no options and retrieve with no options and the two meet. */ readonly corpus?: MemoryIdentity; /** * How to cut the documents. Default `byHeading()` — the one strategy that * reads the document's own structure instead of inferring it. */ readonly splitter?: Splitter; /** Loaders consulted before the built-ins. */ readonly loaders?: readonly DocumentLoader[]; /** * The embedder's id, stored per vector and refused on when it changes. * Defaults to `embedder.id`. */ readonly embedderId?: string; /** Chunks per embed batch. Default 64. */ readonly batchSize?: number; /** * How many batches embed at once. Default 4. * * A hard ceiling on PARALLELISM, not on total work: batches beyond it wait * for the next window rather than being dropped. */ readonly maxConcurrentBatches?: number; /** Attempts per batch, including the first. Default 3. */ readonly attempts?: number; /** * Delete chunks whose document is no longer in the source. Default true — * an index that answers from a document you deleted is worse than one that * does not answer. */ readonly removeMissing?: boolean; /** * The embedder's input ceiling in characters. Chunks longer than this are * embedded anyway (the embedder clips them) and RECORDED in * `report.truncated` / `report.truncatedCount` — so silent half-embedding * becomes a number you can see, and a run that clipped anything says so once * on `console.warn`. * * **Default (9.1.0): the embedder's own declared `maxInputChars`**, falling * back to 2,000 — the measured `localEmbedder` cliff — for an embedder that * declares none. Passing a number here always wins over both: you are * allowed to know your corpus is denser than the embedder's own arithmetic * assumes. * * Before 9.1.0 this defaulted to 2,000 for EVERY embedder, which is the * on-device cliff applied to hosted models that read sixteen times as much. * Combined with a splitter told to cut larger chunks — `byHeading({ maxChars: * 2500 })` — the result was a corpus indexed by the OPENING of each chunk * while the whole chunk was served as the passage, so retrieval could not * find wording plainly visible in the block the model was shown. */ readonly maxChunkChars?: number; } /** The chart's scope. Every field here is in the commit log. */ interface IndexState { discoveredCount: number; documents: readonly LoadedDocument[]; failed: readonly FailedDocument[]; chunks: readonly Chunk[]; toEmbed: readonly Chunk[]; toSkip: readonly string[]; toRemove: readonly string[]; /** The queue of batches still to embed. Drained one window at a time. */ pendingBatches: readonly (readonly Chunk[])[]; /** The window being fanned out over — never longer than `maxConcurrentBatches`. */ window: readonly (readonly Chunk[])[]; batchResults: readonly ({ written?: number; } | undefined)[]; embeddedCount: number; removedCount: number; truncated: readonly TruncatedChunk[]; report?: IndexReport; [key: string]: unknown; } /** * Build and run the indexing chart. * * @returns the report — which is also committed to the run's own log, so it is * evidence rather than only a return value. * * @example * ```ts * const report = await indexCorpus({ * source: { dir: './docs' }, * store: sqliteVectorStore({ file: './corpus.db' }), * embedder: staticEmbedder(), * }); * // { discovered: 3, loaded: 3, chunks: 14, embedded: 14, skipped: 0, removed: 0, … } * ``` */ export declare function indexCorpus(config: IndexCorpusConfig): Promise; /** * The chart itself, exposed so a caller can mount it, attach recorders to it, * or run it under their own executor — the same freedom every other chart in * this library gives. */ export declare function buildIndexChart(config: IndexCorpusConfig): ReturnType; declare function compile(config: IndexCorpusConfig): import("footprintjs").RunnableFlowChart>; export {};