/**
 * Durable ingestion: keeps a vector store, and any keyword indexes beside it, in step with a corpus
 * that changes.
 *
 * A manifest in any `Store` records, for every document, the version of its content, the chunking,
 * the embedding model, and each chunk's version. A run reads the corpus, embeds only the chunks that
 * are new or changed, deletes the chunks a document no longer has and the documents the corpus no
 * longer has, and writes every change idempotently, before the manifest records it. A run that dies
 * halfway leaves a manifest that says exactly what finished; the next run skips it, and inside a
 * durable operation it starts at the document it stopped on.
 */
import type { BatchEmbedder } from '../embeddings/batch.js';
import type { EmbeddingProvider, VectorStore } from '../hallucination/retrieval.js';
import type { OperationContext, OperationExecutor } from '../types/operations.js';
import type { Store } from '../types/store.js';
import { type DocumentSource, type IngestionOptions } from './ingestion.js';
import type { SparseRetriever } from './retrievers.js';
/** A source of documents: a loader, an array, or any iterable, read once per run. */
export type IngestionSource = AsyncIterable<DocumentSource> | Iterable<DocumentSource>;
/** Options for `createIngestionPipeline()`. */
export interface IngestionPipelineOptions {
    /** Names the pipeline. Its manifest is kept under this name, so two pipelines never share one. */
    name: string;
    /**
     * Where the manifest is kept: any `Store`, so it lives in memory, Redis, Postgres, or SQLite. It
     * holds one item per document, and a `MemoryStore` keeps 10,000 items unless given a larger
     * `maxItems`: a manifest that forgets a document re-embeds it.
     */
    manifest: Store;
    /** The vector store chunks are embedded into. */
    vectors: VectorStore;
    /**
     * Keyword indexes kept in step, such as an `ElasticsearchKeywordIndex`. A `PostgresKeywordIndex`
     * that is `shared` needs no place here: the vector store's writes already reach it.
     */
    keywords?: readonly SparseRetriever[];
    /** Embeds chunk texts. Called only for chunks that are new or changed. Needed unless `batchEmbed` is given. */
    embed?: EmbeddingProvider;
    /**
     * Embeds through a provider's batch tier instead, at its batch price: `batchEmbedder()` from
     * `nexus-ai-pro/embeddings/batch`. Chunks to embed are gathered into windows of its `size`, each
     * submitted as one batch, and each window is written once its batch completes, in order. Inside a
     * durable operation every batch is recorded the moment it exists, so a later attempt collects it
     * rather than submitting it again. Taken over `embed` when both are given.
     */
    batchEmbed?: BatchEmbedder;
    /** Provider batches waited on at once, with `batchEmbed`; each holds its window in memory. Defaults to 4. */
    batchesInFlight?: number;
    /** The embedding model, recorded with every document: a different one re-embeds everything. */
    embeddingModel: string;
    /** How documents are split. Different options re-chunk every document. */
    chunking?: IngestionOptions;
    /** Documents read, embedded, and recorded together. Defaults to 64. */
    batchSize?: number;
}
/** Options for one run. */
export interface IngestionRunOptions {
    /** Stops the run between batches. Everything recorded so far stays recorded. */
    signal?: AbortSignal;
    /**
     * Removes the documents the manifest has and this run did not see, with their chunks. Defaults to
     * true: the source is the whole corpus. Turn it off when a run loads only part of it.
     */
    deleteMissing?: boolean;
    /** Called after each batch, with the run so far. */
    onProgress?: (progress: IngestionReport) => void;
    /**
     * The durable operation the run belongs to, when it runs inside an `OperationRunner`. After every
     * batch the run records how far it got with `heartbeat()`, and a later attempt reads it from
     * `previousHeartbeat` and skips straight past what was finished.
     */
    operation?: Pick<OperationContext, 'heartbeat' | 'previousHeartbeat'>;
}
/** What a run did. */
export interface IngestionReport {
    /** Documents read. */
    documents: number;
    /** Documents whose content, chunking, and model were all as recorded, so nothing was done. */
    unchanged: number;
    /** Documents the manifest did not have. */
    added: number;
    /** Documents that changed. */
    updated: number;
    /** Documents removed because the corpus no longer has them. */
    deleted: number;
    /** Chunks embedded and written. */
    chunksEmbedded: number;
    /** Chunks of changed documents whose text did not change, so their embedding was kept. */
    chunksKept: number;
    /** Chunks removed. */
    chunksDeleted: number;
    /** Documents a resumed run skipped because an earlier attempt had finished them. */
    resumed: number;
    /** Provider batches submitted, with `batchEmbed`. */
    batches?: number;
    /** Provider batches an earlier attempt submitted, which this run collected instead of submitting again. */
    batchesRecovered?: number;
}
/** What the manifest records for one document. */
export interface IngestedDocument {
    /** The document's id. */
    id: string;
    /** A hash of its text, source, and metadata. */
    version: string;
    /** A hash of the chunking options it was split with. */
    chunking: string;
    /** The embedding model its chunks were embedded with. */
    model: string;
    /** Each chunk's id, with the hash of its text: the chunk's version. */
    chunks: Record<string, string>;
    /** When it was last written, as ISO-8601. */
    updatedAt: string;
}
/** A pipeline: run it as often as the corpus changes. */
export interface IngestionPipeline {
    /** Brings the stores in step with the documents a source yields. */
    run(source: IngestionSource | readonly IngestionSource[], options?: IngestionRunOptions): Promise<IngestionReport>;
    /** What the manifest records for a document, or `undefined`. */
    document(id: string): Promise<IngestedDocument | undefined>;
    /** Removes documents by id, with their chunks, from every store and the manifest. */
    remove(ids: readonly string[]): Promise<number>;
}
/**
 * Builds a pipeline that keeps the stores in step with a corpus, re-embedding only what changed.
 *
 * Documents need an `id` (or a `source`), which is what ties a document to its earlier version. A
 * chunk's id is its document's id and a hash of its text, so a chunk whose text did not change keeps
 * its embedding wherever it moved. Chunks are cut by length, within each Markdown section when
 * `splitOnMarkdownHeadings` is on, so an edit re-embeds the chunks of its section from the edit on,
 * and a section added or removed leaves the others alone. One run at a time per pipeline; an
 * operation's idempotency key enforces that across workers.
 *
 * ```ts
 * const pipeline = createIngestionPipeline({
 *   name: 'help-center',
 *   manifest: store,
 *   vectors: new PostgresVectorStore(pool, { dimensions: 1536 }),
 *   embed,
 *   embeddingModel: 'text-embedding-3-small',
 * });
 * const report = await pipeline.run(loadDirectory('./docs'));
 * ```
 */
export declare function createIngestionPipeline(options: IngestionPipelineOptions): IngestionPipeline;
/**
 * The pipeline as the executor of a durable operation: submit it to an `OperationRunner`, and an
 * attempt that dies halfway is recovered by another worker, which starts at the document the first
 * one stopped on.
 *
 * ```ts
 * const runner = new OperationRunner({ store: new RedisOperationStore(redis), retry: { maxAttempts: 5 } });
 * await runner.submit(ingestionExecutor(pipeline, () => loadDirectory('./docs')), { idempotencyKey: 'help-center' });
 * ```
 */
export declare function ingestionExecutor(pipeline: IngestionPipeline, source: () => IngestionSource | readonly IngestionSource[], options?: Omit<IngestionRunOptions, 'operation' | 'signal'>): OperationExecutor<IngestionReport>;