/** * indexDocuments — seed a vector-capable MemoryStore with documents. * * Embeds each document, builds a `MemoryEntry<{content, metadata?}>`, * batches into `store.putMany()`. Used at application startup to * populate a RAG store before the first agent run. * * Pattern: Bulk-write helper. Not a flowchart stage — it runs once * at boot, not per-iteration. * Role: Layer-3 RAG pipeline starter. Pairs with `defineRAG()` * which only does the read side. * Emits: N/A — startup-time batch write, not part of the agent run. * * @example * ```ts * import { Agent, indexDocuments, defineRAG } from 'agentfootprint' import { InMemoryStore, mockEmbedder } from 'agentfootprint/memory'; * * const store = new InMemoryStore(); * const embedder = mockEmbedder(); * * await indexDocuments(store, embedder, [ * { id: 'doc1', content: 'Refunds processed within 3 business days.' }, * { id: 'doc2', content: 'Pro plan: $20/mo, includes priority support.', metadata: { tier: 'pro' } }, * { id: 'doc3', content: 'Free plan: limited to 100 calls/month.' }, * ]); * * const docs = defineRAG({ id: 'product-docs', store, embedder }); * const agent = Agent.create({ provider }).rag(docs).build(); * ``` */ import type { Embedder } from '../../memory/embedding/index.js'; import type { MemoryStore } from '../../memory/store/index.js'; import type { MemoryIdentity } from '../../memory/identity/index.js'; /** * A document to index. `id` must be unique within the store + identity. * * The passage rides on `content`. Since 8.19.0 `text` is accepted as the * same thing, because the retrieval formatter reads both keys and a * hand-built entry that spelled it `text` used to index and retrieve * perfectly while rendering an EMPTY passage. One sane meaning, two * spellings, and `indexDocuments` refuses a document carrying NEITHER — * an unrenderable passage and an absent one are different facts. */ export type RagDocument = { readonly id: string; /** The passage. Use this one; `text` is the accepted alias. */ readonly content: string; readonly text?: string; readonly metadata?: Readonly>; } | { readonly id: string; readonly content?: string; /** * The passage, spelled the way a `Chunk` spells it. Accepted so a * corpus assembled by hand from `rag`-door chunks indexes without a * rename. `content` wins when both are present. */ readonly text: string; readonly metadata?: Readonly>; }; export interface IndexDocumentsOptions { /** * Identity scope to write under. Default: a single shared * `{ conversationId: '_global' }` namespace, suitable for app-wide * corpora. * * **Multi-tenant footgun:** the read side (`agent.run({ identity })`) * queries within whichever identity is passed at request time. * If you index here under `_global` but query under * `{ tenant: 'acme' }`, you'll get ZERO results — silently. Either: * 1. Index every document under each tenant's identity (duplicated * storage, but isolated), or * 2. Index under `_global` AND query under `_global` (shared * corpus across tenants — fine for product docs, NOT for * tenant-private data), or * 3. Use a vector store adapter that supports multi-namespace * reads at query time (Pinecone, Qdrant — outside this helper's * scope). */ readonly identity?: MemoryIdentity; /** * Stable id of the embedder. Stored on each entry so a future embedder swap * doesn't silently mix similarity scores. * * Defaults to the embedder's own `id` (8.9.0 — every shipped embedder sets * one), and to `'default-embedder'` for a hand-written `Embedder` that has * none. That default matters more with a durable store: it is half of the * `'@'` fingerprint `sqliteVectorStore` records per vector and * refuses on, so an index built by `staticEmbedder()` will not silently * accept vectors from `openaiEmbedder()`. */ readonly embedderId?: string; /** * Optional tier tag to attach to indexed entries (`'hot'` / * `'warm'` / `'cold'`). Useful when read-side `defineRAG` should * filter to a subset of the corpus. */ readonly tier?: 'hot' | 'warm' | 'cold'; /** * Optional TTL in milliseconds from indexing time. Useful for * compliance retention windows (e.g., re-index quarterly). */ readonly ttlMs?: number; /** * Optional abort signal — embedders making network calls thread * this through to abort batch indexing on shutdown / timeout. */ readonly signal?: AbortSignal; /** * Called once with the cost of the embedding work this call did (8.9.0). * * `indexDocuments` runs at STARTUP, outside any agent run, so it has no * emit channel to ride — there is no scope, no dispatcher and no * `runtimeStageId` to correlate against. Rather than pretend otherwise, it * hands the same payload the in-run stages emit as * `agentfootprint.embedding.generated` straight to you, so the index-time * half of the cost model is reportable from a boot script: * * ```ts * await indexDocuments(store, embedder, docs, { * onEmbedding: (e) => console.log(`embedded ${e.count} documents in ${e.durationMs}ms`), * }); * ``` */ readonly onEmbedding?: (payload: import('../../events/payloads.js').EmbeddingGeneratedPayload) => void; /** * Max number of concurrent embed calls when the embedder doesn't * implement `embedBatch`. Default `8`. Without this cap, a 10K-doc * corpus would fire 10K parallel embed calls and trigger rate limits. * Ignored when `embedBatch` is available (the embedder controls * its own batching). */ readonly maxConcurrency?: number; /** * The embedder's input ceiling in characters (9.1.0). Documents longer than * it are embedded anyway — the backend CLIPS them rather than refusing — and * the run says so once on `console.warn`. * * Default: the embedder's own declared `maxInputChars`, falling back to * 2,000 (the measured `localEmbedder` cliff) for one that declares none. * An explicit number here wins over both. * * This helper does not split — a document you hand it is stored whole and * served whole. So a document past the ceiling is indexed by its OPENING * while the model is later shown all of it, and retrieval cannot find * wording that is plainly there. Nothing throws; the corpus is quietly * partially indexed. Cut such documents into chunks first — the * `agentfootprint/rag` door does exactly that, and its chunks keep the * coordinates a citation is checked against. */ readonly maxChunkChars?: number; } /** * Embed + persist documents. Returns the count actually indexed * (skips duplicates if the store rejects them). Throws on embedder * failure or store error — fail loud at startup is desirable. * * **Re-indexing semantics:** entries are written with `version: 1` and * `putMany` (most adapters: last-write-wins). Re-running this helper * after the store has been mutated by other writers may stomp their * versions. For idempotent corpus refresh, either delete-then-index * or use a custom upsert via `store.putIfVersion()` per document. A * first-class `mode: 'upsert' | 'replace'` API is planned for a * future release. */ export declare function indexDocuments(store: MemoryStore, embedder: Embedder, documents: readonly RagDocument[], options?: IndexDocumentsOptions): Promise;