/**
 * BM25 (Best Matching 25) implementation
 * Using StarCoder2 tokenizer for code-aware tokenization
 *
 * BM25 improves on TF-IDF with:
 * 1. Term frequency saturation (k1 parameter) - diminishing returns for repeated terms
 * 2. Document length normalization (b parameter) - adjusts for document length
 */
import { initializeTokenizer } from './code-tokenizer.js';
export { initializeTokenizer };
export interface DocumentVector {
    uri: string;
    terms: Map<string, number>;
    rawTerms: Map<string, number>;
    magnitude: number;
}
export interface SearchIndex {
    documents: DocumentVector[];
    idf: Map<string, number>;
    totalDocuments: number;
    metadata: {
        generatedAt: string;
        version: string;
    };
}
/**
 * Tokenize code using StarCoder2 (async)
 */
export declare function tokenize(text: string): Promise<string[]>;
/**
 * Build TF-IDF search index from documents (async - uses StarCoder2)
 */
export declare function buildSearchIndex(documents: Array<{
    uri: string;
    content: string;
}>): Promise<SearchIndex>;
/**
 * Calculate cosine similarity between query and document
 */
export declare function calculateCosineSimilarity(queryVector: Map<string, number>, docVector: DocumentVector): number;
/**
 * Process query into TF-IDF vector (async - uses StarCoder2)
 */
export declare function processQuery(query: string, idf: Map<string, number>): Promise<Map<string, number>>;
/**
 * SQL-based search result from storage
 * Uses pre-computed magnitude and token count for BM25 scoring
 */
export interface StorageSearchResult {
    path: string;
    matchedTerms: Map<string, {
        tfidf: number;
        rawFreq: number;
    }>;
    magnitude: number;
    tokenCount: number;
}
/**
 * Search documents using BM25 scoring (SQL-based storage)
 *
 * BM25 formula: score(D,Q) = Σ IDF(qi) * (f(qi,D) * (k1+1)) / (f(qi,D) + k1 * (1 - b + b * |D|/avgdl))
 *
 * Where:
 * - f(qi,D) = raw frequency of term qi in document D
 * - |D| = document length (token count)
 * - avgdl = average document length
 * - k1 = term frequency saturation (default: 1.2)
 * - b = length normalization (default: 0.75)
 */
export declare function searchDocumentsFromStorage(query: string, candidates: StorageSearchResult[], idf: Map<string, number>, options?: {
    limit?: number;
    minScore?: number;
    avgDocLength?: number;
}): Promise<Array<{
    uri: string;
    score: number;
    matchedTerms: string[];
}>>;
/**
 * Get query tokens (exported for SQL-based search) - async
 */
export declare function getQueryTokens(query: string): Promise<string[]>;
/**
 * Search documents using BM25 scoring (in-memory index)
 *
 * For in-memory search, document length is calculated from rawTerms.
 * Average document length is calculated from all documents in the index.
 */
export declare function searchDocuments(query: string, index: SearchIndex, options?: {
    limit?: number;
    minScore?: number;
}): Promise<Array<{
    uri: string;
    score: number;
    matchedTerms: string[];
}>>;
//# sourceMappingURL=tfidf.d.ts.map