import { DyNTS_LVS_BM25_Corpus, DyNTS_LVS_BM25_DocScore, dyNTS_LVS_BM25_minMaxNormalize, } from './lvs-bm25.util'; describe('| DyNTS_LVS_BM25_Corpus.tokenize', (): void => { it('| lowercase + split on \\w+ boundaries', (): void => { expect(DyNTS_LVS_BM25_Corpus.tokenize('Hello, World!')).toEqual(['hello', 'world']); }); it('| identifier marad egy tokenkent (PascalCase)', (): void => { expect(DyNTS_LVS_BM25_Corpus.tokenize('UserController')).toEqual(['usercontroller']); }); it('| hyphenated nev ket tokenre esik', (): void => { expect(DyNTS_LVS_BM25_Corpus.tokenize('auth-flow')).toEqual(['auth', 'flow']); }); it('| ures string → ures array', (): void => { expect(DyNTS_LVS_BM25_Corpus.tokenize('')).toEqual([]); }); it('| csak whitespace/punctuation → ures array', (): void => { expect(DyNTS_LVS_BM25_Corpus.tokenize(' ,.! ')).toEqual([]); }); it('| underscore megmarad (snake_case is egy token)', (): void => { expect(DyNTS_LVS_BM25_Corpus.tokenize('user_controller')).toEqual(['user_controller']); }); }); describe('| DyNTS_LVS_BM25_Corpus.score', (): void => { it('| ures corpus → ures array', (): void => { const corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus([]); expect(corpus.size()).toBe(0); expect(corpus.score('anything')).toEqual([]); }); it('| ures query → minden doc 0 score-t kap', (): void => { const corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus([ { id: 'a', text: 'foo bar' }, { id: 'b', text: 'baz' }, ]); const out: DyNTS_LVS_BM25_DocScore[] = corpus.score(''); expect(out.length).toBe(2); expect(out.every((s: DyNTS_LVS_BM25_DocScore) => s.score === 0)).toBe(true); }); it('| query NEM matchelo termmel → minden score 0', (): void => { const corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus([ { id: 'a', text: 'foo bar' }, { id: 'b', text: 'baz' }, ]); const out: DyNTS_LVS_BM25_DocScore[] = corpus.score('xyz'); expect(out.every((s: DyNTS_LVS_BM25_DocScore) => s.score === 0)).toBe(true); }); it('| relevant doc magasabb score-t kap mint nem-relevant', (): void => { const corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus([ { id: 'a', text: 'the UserController handles authentication' }, { id: 'b', text: 'cooking recipes for desserts' }, { id: 'c', text: 'database setup guide' }, ]); const out: DyNTS_LVS_BM25_DocScore[] = corpus.score('UserController'); const a: DyNTS_LVS_BM25_DocScore = out.find((s) => s.id === 'a')!; const b: DyNTS_LVS_BM25_DocScore = out.find((s) => s.id === 'b')!; expect(a.score).toBeGreaterThan(0); expect(b.score).toBe(0); expect(a.score).toBeGreaterThan(b.score); }); it('| rarer term (alacsonyabb df) magasabb IDF-et ad → magasabb score', (): void => { // 'common' minden docban → alacsony IDF; 'rare' csak az 'a' docban → magas IDF const corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus([ { id: 'a', text: 'common rare' }, { id: 'b', text: 'common' }, { id: 'c', text: 'common' }, { id: 'd', text: 'common' }, ]); const commonOut: DyNTS_LVS_BM25_DocScore[] = corpus.score('common'); const rareOut: DyNTS_LVS_BM25_DocScore[] = corpus.score('rare'); const aCommon: number = commonOut.find((s) => s.id === 'a')!.score; const aRare: number = rareOut.find((s) => s.id === 'a')!.score; expect(aRare).toBeGreaterThan(aCommon); }); it('| query-term ismetles NEM no monotonan a score-on (term saturation k1)', (): void => { // 'a' egy 'foo'-t tartalmaz, 'b' tobbet const corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus([ { id: 'a', text: 'foo bar baz qux' }, { id: 'b', text: 'foo foo foo foo bar baz qux' }, ]); const out: DyNTS_LVS_BM25_DocScore[] = corpus.score('foo'); const a: number = out.find((s) => s.id === 'a')!.score; const b: number = out.find((s) => s.id === 'b')!.score; // b > a, de NEM 4x a-szor (saturation) expect(b).toBeGreaterThan(a); expect(b).toBeLessThan(4 * a); }); it('| case-insensitive — uppercase query matchel lowercase doc-ot', (): void => { const corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus([ { id: 'a', text: 'usercontroller is great' }, ]); const out: DyNTS_LVS_BM25_DocScore[] = corpus.score('USERCONTROLLER'); expect(out[0].score).toBeGreaterThan(0); }); it('| ket query term aggregalja az IDF-eket additivan', (): void => { const corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus([ { id: 'a', text: 'auth flow handler' }, { id: 'b', text: 'auth only' }, { id: 'c', text: 'flow only' }, ]); const both: number = corpus.score('auth flow').find((s) => s.id === 'a')!.score; const authOnly: number = corpus.score('auth').find((s) => s.id === 'a')!.score; expect(both).toBeGreaterThan(authOnly); }); it('| invalid input doc (null/missing id/text) silently skipd', (): void => { const corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus([ { id: 'a', text: 'hello' }, // eslint-disable-next-line @typescript-eslint/no-explicit-any null as any, // eslint-disable-next-line @typescript-eslint/no-explicit-any { id: 'b' } as any, ]); expect(corpus.size()).toBe(1); }); }); describe('| dyNTS_LVS_BM25_minMaxNormalize', (): void => { it('| ures input → ures array', (): void => { expect(dyNTS_LVS_BM25_minMaxNormalize([])).toEqual([]); }); it('| min 0, max 10 → normalizalva [0..1]', (): void => { const out: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize([ { id: 'a', score: 0 }, { id: 'b', score: 5 }, { id: 'c', score: 10 }, ]); expect(out.find((s) => s.id === 'a')!.score).toBeCloseTo(0, 5); expect(out.find((s) => s.id === 'b')!.score).toBeCloseTo(0.5, 5); expect(out.find((s) => s.id === 'c')!.score).toBeCloseTo(1, 5); }); it('| azonos score → minden 0 (NEM 0.5, nincs jel diszkriminaciora)', (): void => { const out: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize([ { id: 'a', score: 3 }, { id: 'b', score: 3 }, ]); expect(out.every((s) => s.score === 0)).toBe(true); }); });