/** * BM25 text-search ranking util a LVS hybrid search-hez (FR-004). * * Pure TypeScript, dependency-free. In-memory corpus alapjan szamol score-okat, * NEM perzisztal indexet (a hybrid hivasonkent ujraepiti a corpus-t a candidate * dokumentumokon — kis (~100..10000 dokumentum) LVS-corpus eseten ez gyors es * egyszeru). * * Canonical params: k1=1.2, b=0.75 (industry standard a "lucene-szeru" * implementaciokban). NEM expose-oltak — ha kell, FR-002 kovetkezo iteracioban * tehetjuk parameterizalhatova. * * Tokenizer: `text.toLowerCase().match(/\w+/g) || []`. Case-insensitive, * alphanumeric+underscore boundary-k. `UserController` egy token marad (jo az * identifier match-re), `auth-flow` ket tokenre esik (auth + flow). * * IDF formula (BM25+): `log((N - df + 0.5) / (df + 0.5) + 1)`. A +1 a logon * belul garantalja, hogy a kozos szavak is pozitiv (kicsi) IDF-et kapjanak, * NEM negativ-t — fontos a hybrid score-merge-nel hogy ne huzzon le dokumentumot * ahol kozos szo szerepel. */ /** BM25 k1 parameter (term saturation control). Canonical default. */ const BM25_K1: number = 1.2; /** BM25 b parameter (length normalization weight, 0=off, 1=full). Canonical default. */ const BM25_B: number = 0.75; /** Token regex — alphanumeric + underscore. */ const TOKEN_REGEX: RegExp = /\w+/g; /** * Egy dokumentum BM25-score-ja egy query ellen, egy elore-felepitett corpus * konteztusaban. */ export interface DyNTS_LVS_BM25_DocScore { /** Dokumentum azonosito (mint az LVS_SearchResult `id`-jaben). */ id: string; /** Nyers BM25 score (0..∞). NEM normalizalt. */ score: number; } /** * Felepitett BM25 corpus — egy adott dokumentumhalmaz indexe. A `score()` az * indexen kerdez le egy query-t es minden dokumentumra ad egy score-t. * * Egy corpus egyszer-hasznalatos a hybrid search hivasban — NEM kell cache-elni, * a felepites O(N * |doc|) ami pici N-re elhanyagolhato. */ export class DyNTS_LVS_BM25_Corpus { /** Tokenizalt dokumentumok: id -> tokens. */ private readonly docTokens: Map = new Map(); /** Doc-length: id -> token count. */ private readonly docLengths: Map = new Map(); /** Term -> doc-frequency (hany docban szerepel az adott term, legalabb 1x). */ private readonly termDocFreq: Map = new Map(); /** Term -> id -> term-frequency a docban. */ private readonly termFreqByDoc: Map> = new Map>(); /** Atlagos dokumentum-hossz (token count). */ private avgDocLength: number = 0; /** Total doc count. */ private docCount: number = 0; /** * Letrehoz egy uj corpus-t a megadott id->text parok-bol. * NEM dob hibat ures input-ra — ures corpus ervenyes (minden score = 0). */ constructor(docs: { id: string; text: string }[]) { if (!Array.isArray(docs) || docs.length === 0) { return; } let totalLength: number = 0; for (const doc of docs) { if (!doc || typeof doc.id !== 'string' || typeof doc.text !== 'string') { continue; } const tokens: string[] = DyNTS_LVS_BM25_Corpus.tokenize(doc.text); this.docTokens.set(doc.id, tokens); this.docLengths.set(doc.id, tokens.length); totalLength += tokens.length; // Term-frequency a docban const localTf: Map = new Map(); for (const tok of tokens) { localTf.set(tok, (localTf.get(tok) ?? 0) + 1); } for (const [term, tf] of localTf) { // Doc-frequency: +1 per term, per doc this.termDocFreq.set(term, (this.termDocFreq.get(term) ?? 0) + 1); // Term-freq-by-doc reverse index let perDoc: Map | undefined = this.termFreqByDoc.get(term); if (!perDoc) { perDoc = new Map(); this.termFreqByDoc.set(term, perDoc); } perDoc.set(doc.id, tf); } } this.docCount = this.docTokens.size; this.avgDocLength = this.docCount > 0 ? totalLength / this.docCount : 0; } /** * Public tokenizer — exportalt, hogy spec-ek + hivok ugyanazt a normalizalast * tudjak hasznalni mint a corpus. */ static tokenize(text: string): string[] { if (typeof text !== 'string' || text.length === 0) { return []; } return text.toLowerCase().match(TOKEN_REGEX) ?? []; } /** * Visszaadja a corpus dokumentum-szamat (NEM-ures docok). */ size(): number { return this.docCount; } /** * BM25 score minden dokumentumra a query-re. * * Ures query → minden score 0 (degenerate case; a hivo kezelje ha kell). * Ures corpus → ures array. */ score(query: string): DyNTS_LVS_BM25_DocScore[] { if (this.docCount === 0) { return []; } const queryTokens: string[] = DyNTS_LVS_BM25_Corpus.tokenize(query); if (queryTokens.length === 0) { // Minden doc 0 score-t kap const results: DyNTS_LVS_BM25_DocScore[] = []; for (const id of this.docTokens.keys()) { results.push({ id: id, score: 0 }); } return results; } // Egyedi query-termek halmaza (ismetlodes nem ad nagyobb IDF-et) const uniqueTerms: string[] = Array.from(new Set(queryTokens)); // Pre-compute IDF a query-termekre const idfMap: Map = new Map(); for (const term of uniqueTerms) { const df: number = this.termDocFreq.get(term) ?? 0; // BM25+ IDF: log((N - df + 0.5) / (df + 0.5) + 1) const idf: number = Math.log((this.docCount - df + 0.5) / (df + 0.5) + 1); idfMap.set(term, idf); } // Per-doc BM25 score const results: DyNTS_LVS_BM25_DocScore[] = []; for (const [docId, docLen] of this.docLengths) { let score: number = 0; for (const term of uniqueTerms) { const tf: number = this.termFreqByDoc.get(term)?.get(docId) ?? 0; if (tf === 0) { continue; } const idf: number = idfMap.get(term) ?? 0; const norm: number = 1 - BM25_B + BM25_B * (docLen / (this.avgDocLength || 1)); score += idf * (tf * (BM25_K1 + 1)) / (tf + BM25_K1 * norm); } results.push({ id: docId, score: score }); } return results; } } /** * Min-max normalizalas [0,1] tartomanyra. A hybrid score-merge-hez kell a BM25 * score-okat a candidate-szetten 0..1 sav-ba hozni (a cosine mar 0..1). * * Edge case-ek: * - Ures array → []. * - Minden score azonos (max-min === 0) → minden 0.0 (NEM 0.5 vagy 1.0; ha * nincs diszkriminacio, ne tegyunk hozza signal-t). * - Negativ score-ok (BM25+IDF garantal pozitivat, de defensive): a min-max * ugyanugy mukodik. */ export function dyNTS_LVS_BM25_minMaxNormalize( scores: DyNTS_LVS_BM25_DocScore[], ): DyNTS_LVS_BM25_DocScore[] { if (!Array.isArray(scores) || scores.length === 0) { return []; } let min: number = Infinity; let max: number = -Infinity; for (const s of scores) { if (s.score < min) { min = s.score; } if (s.score > max) { max = s.score; } } const range: number = max - min; if (range === 0) { // Nincs diszkriminacio — minden 0.0 (NEM huzzon le, NEM huzzon fel) return scores.map((s: DyNTS_LVS_BM25_DocScore): DyNTS_LVS_BM25_DocScore => ({ id: s.id, score: 0, })); } return scores.map((s: DyNTS_LVS_BM25_DocScore): DyNTS_LVS_BM25_DocScore => ({ id: s.id, score: (s.score - min) / range, })); }