import type { RagChunk } from '../hallucination/rag.js';
import { type DocumentSource, type IngestionOptions } from '../rag/ingestion.js';
export type { DocumentSource } from '../rag/ingestion.js';
/**
 * Documents produced one at a time. Every loader returns one, so a corpus larger than memory streams
 * through ingestion instead of being read whole, and an array of documents works as well.
 */
export type DocumentLoader = AsyncIterable<DocumentSource> | Iterable<DocumentSource>;
/**
 * A file to load: a path, a `file:` URL, content already in memory with a name for it, or a stream of
 * it, such as a response's body or an object store's download. In-memory content is how a loader reads
 * an upload, a database blob, or a file on a runtime without a disk; a stream, how the CSV and JSON
 * loaders read a file larger than memory, as it arrives.
 */
export type FileInput = string | URL | {
    source: string;
    content: string | Uint8Array;
} | {
    source: string;
    stream: AsyncIterable<Uint8Array | string> | Iterable<Uint8Array | string>;
};
/** Reads every document from one or more loaders, in order, into an array. */
export declare function collectDocuments(...loaders: DocumentLoader[]): Promise<DocumentSource[]>;
/**
 * Where loaded chunks go: a vector store, a `KeywordIndex`, or anything else with an `add()` for
 * chunks. Loading into several at once fills a vector store and the keyword index hybrid search reads
 * in one pass over the corpus.
 */
export interface ChunkSink {
    /** Adds chunks, replacing those whose id is already held. */
    add(chunks: RagChunk[]): unknown;
}
/** Options for `loadIntoStore()`: how documents are split, and how many are embedded at once. */
export interface LoadIntoStoreOptions extends IngestionOptions {
    /** Documents split and added to the store together. Defaults to 32. */
    batchSize?: number;
    /** Stops loading between batches. */
    signal?: AbortSignal;
    /** Called after each batch is stored, for progress reporting. */
    onBatch?: (progress: LoadIntoStoreResult) => void;
}
/** What `loadIntoStore()` stored. */
export interface LoadIntoStoreResult {
    /** Documents read from the loaders. */
    documents: number;
    /** Chunks added to the store. */
    chunks: number;
    /** Characters across every document. */
    totalCharacters: number;
}
/**
 * Streams documents from loaders into one or more stores: each batch is split into chunks and added to
 * every store at once, so memory holds one batch at a time however large the corpus is. Chunk ids
 * derive from document ids, so loading the same corpus again replaces its chunks rather than
 * duplicating them. A document with no id is numbered in load order.
 */
export declare function loadIntoStore(stores: ChunkSink | readonly ChunkSink[], loaders: DocumentLoader | readonly DocumentLoader[], options?: LoadIntoStoreOptions): Promise<LoadIntoStoreResult>;