/** * Tests for `substrate/sim.ts` — hybrid dense + sparse similarity over the * v2 concept-page collection. * * The embedding backend and the underlying `@qdrant/js-client-rest` are both * mocked so the test is hermetic and fast. We mock at the Qdrant client * level (not at `../qdrant.js`) so the real `hybridQueryConceptPages` * implementation runs end-to-end — that way nothing about the v2 qdrant * module's exports leaks into other test files in the same Bun process. * * Coverage: * - clamp01 boundaries. * - simBatch fusion math (`dense_weight`, `sparse_weight`, * sparse-batch normalization, [0,1] clamp). * - The Qdrant query is filtered to the candidate slugs. * - Empty candidate list short-circuits without backend calls. */ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; import type { AssistantConfig } from "../../../../../config/types.js"; // --------------------------------------------------------------------------- // Module-level mocks (registered before `await import("../sim.js")`). // --------------------------------------------------------------------------- // Same partial-mock pattern as for the embedding backend: re-export the // real symbols and override only `resolveQdrantUrl` so the v2 qdrant // client picks up our test URL. const realQdrantClient = await import("../../../../../persistence/embeddings/qdrant-client.js"); mock.module("../../../../../persistence/embeddings/qdrant-client.js", () => ({ ...realQdrantClient, resolveQdrantUrl: () => "http://127.0.0.1:6333", })); const state = { embedCalls: [] as Array<{ inputs: unknown[] }>, sparseCalls: [] as string[], embedReturn: [[0.1, 0.2, 0.3]] as number[][], dimensionAvailable: true, // Programmable Qdrant query response — one entry per `using` channel, // shifted in order so each test can stage dense + sparse results. queryResponses: { dense: [] as Array<{ points: Array<{ score?: number; payload: Record }>; }>, sparse: [] as Array<{ points: Array<{ score?: number; payload: Record }>; }>, }, queryCalls: [] as Array<{ collection: string; using: string; limit: number; filter: unknown; }>, }; // Re-export every real symbol from the embedding-backend module, overriding // only the one we control. Bun's `mock.module` replacement is process-wide, // so a partial mock here would break sibling test files that import other // exports from the same module (`selectEmbeddingBackend`, etc.). const realEmbeddingBackend = await import("../../../../../persistence/embeddings/embedding-backend.js"); mock.module( "../../../../../persistence/embeddings/embedding-backend.js", () => ({ ...realEmbeddingBackend, isEmbeddingDimensionAvailable: async () => state.dimensionAvailable, embedWithBackend: async (_config: AssistantConfig, inputs: unknown[]) => { state.embedCalls.push({ inputs }); return { provider: "local", model: "test-model", vectors: state.embedReturn, }; }, }), ); // `sim.ts` builds the query-side sparse vector via BM25's // `generateBm25QueryEmbedding`. Wrap it to record the call text, then // delegate to the real implementation so the resulting sparse vector is // well-formed. Capture the function reference *before* registering the // mock — ESM live bindings resolve through the namespace at call time, so // `realSparseBm25.fn(...)` after `mock.module` would route into the // mocked version and recurse. const realSparseBm25 = await import("../sparse-bm25.js"); const realGenerateBm25QueryEmbedding = realSparseBm25.generateBm25QueryEmbedding; mock.module("../sparse-bm25.js", () => ({ ...realSparseBm25, generateBm25QueryEmbedding: (text: string) => { state.sparseCalls.push(text); return realGenerateBm25QueryEmbedding(text); }, })); class MockQdrantClient { constructor(_opts: unknown) {} async collectionExists(_name: string) { // The v2 qdrant module's readiness cache latches on the first call, // so reporting "exists: true" once is enough for every test. return { exists: true }; } async createCollection() { return {}; } async createPayloadIndex() { return {}; } async query( name: string, params: { using: string; limit: number; filter?: unknown }, ) { state.queryCalls.push({ collection: name, using: params.using, limit: params.limit, filter: params.filter, }); // Both `dense` and `summary_dense` consume from the dense queue (and // similarly for sparse). The four-channel hybrid query fires them in // order: body-dense, body-sparse, summary-dense, summary-sparse — so // the queue order matches the call order. const channel = params.using.endsWith("sparse") ? "sparse" : "dense"; return state.queryResponses[channel].shift() ?? { points: [] }; } } mock.module("@qdrant/js-client-rest", () => ({ QdrantClient: MockQdrantClient, })); const { simBatch, clamp01, effectiveWeights } = await import("../sim.js"); const { _resetMemoryV2QdrantForTests } = await import("../qdrant.js"); // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- function resetState(): void { state.embedCalls.length = 0; state.sparseCalls.length = 0; state.embedReturn = [[0.1, 0.2, 0.3]]; state.dimensionAvailable = true; state.queryResponses.dense.length = 0; state.queryResponses.sparse.length = 0; state.queryCalls.length = 0; // Bun's `mock.module` persists across files in the same process, so the // qdrant module's singleton may already hold a MockQdrantClient instance // from a sibling test file. Reset readiness so each test in this file // gets a fresh `new QdrantClient()` resolved against our mock. _resetMemoryV2QdrantForTests(); } function configWithWeights( denseWeight: number, sparseWeight: number, ): AssistantConfig { // Only the fields sim.ts touches are populated; the rest of AssistantConfig // is irrelevant here because `embedWithBackend` is mocked. return { memory: { v2: { dense_weight: denseWeight, sparse_weight: sparseWeight, }, }, } as unknown as AssistantConfig; } /** * Stage a single Qdrant response that maps each (slug, denseScore?, sparseScore?) * tuple onto the dense or sparse channel, mirroring how `hybridQueryConceptPages` * merges per-channel hits. Optional `summaryDenseScore` / `summarySparseScore` * stage the summary-side channels — pages without those entries fall through * to body-only scoring at fusion time. */ function stageHybridResponse( hits: Array<{ slug: string; denseScore?: number; sparseScore?: number; summaryDenseScore?: number; summarySparseScore?: number; }>, ): void { state.queryResponses.dense.push({ points: hits .filter((h) => h.denseScore !== undefined) .map((h) => ({ score: h.denseScore, payload: { slug: h.slug } })), }); state.queryResponses.sparse.push({ points: hits .filter((h) => h.sparseScore !== undefined) .map((h) => ({ score: h.sparseScore, payload: { slug: h.slug } })), }); // The four-channel hybrid query also fires `summary_dense` and // `summary_sparse` queries against the same collection. Tests that don't // care about summary scores leave those channels empty so the fallback // (body-only) path runs. state.queryResponses.dense.push({ points: hits .filter((h) => h.summaryDenseScore !== undefined) .map((h) => ({ score: h.summaryDenseScore, payload: { slug: h.slug } })), }); state.queryResponses.sparse.push({ points: hits .filter((h) => h.summarySparseScore !== undefined) .map((h) => ({ score: h.summarySparseScore, payload: { slug: h.slug } })), }); } beforeEach(resetState); afterEach(resetState); // --------------------------------------------------------------------------- // clamp01 // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- // effectiveWeights — adaptive sparse weighting // --------------------------------------------------------------------------- describe("effectiveWeights", () => { // The helper takes a generic config, but only reads // `memory.v2.min_sparse_spread` / `full_sparse_spread`. Build a minimal // shape so test cases can opt into custom thresholds vs the built-in // defaults (0.2 / 0.5). function configWithSpreadOverrides( min?: number, full?: number, ): AssistantConfig { return { memory: { v2: { min_sparse_spread: min, full_sparse_spread: full } }, } as unknown as AssistantConfig; } const baseConfig = configWithSpreadOverrides(); test("returns base weights when sparse weight is zero", () => { const result = effectiveWeights( [{ sparseScore: 1 }, { sparseScore: 2 }], 2, 1.0, 0.0, baseConfig, ); expect(result.dense).toBe(1.0); expect(result.sparse).toBe(0); }); test("returns base weights when fewer than 2 sparse-bearing hits", () => { // Single sparse-bearing hit — spread is undefined. const result = effectiveWeights( [{ sparseScore: 5 }, {}], 5, 0.7, 0.3, baseConfig, ); expect(result.dense).toBeCloseTo(0.7); expect(result.sparse).toBeCloseTo(0.3); expect(result.spread).toBe(0); }); test("collapses sparse weight to 0 when spread is below min_sparse_spread", () => { // Three hits with sparseNorm = {0.95, 0.97, 1.0} → spread 0.05 < 0.2. const hits = [ { sparseScore: 9.5 }, { sparseScore: 9.7 }, { sparseScore: 10 }, ]; const result = effectiveWeights(hits, 10, 0.7, 0.3, baseConfig); expect(result.spread).toBeCloseTo(0.05, 6); expect(result.sparse).toBeCloseTo(0, 6); // Dense compensates: gets the full sparse weight added back. expect(result.dense).toBeCloseTo(1.0, 6); }); test("preserves base weights when spread reaches full_sparse_spread", () => { // sparseNorm = {0.5, 1.0} → spread 0.5 === default full threshold. const hits = [{ sparseScore: 5 }, { sparseScore: 10 }]; const result = effectiveWeights(hits, 10, 0.7, 0.3, baseConfig); expect(result.spread).toBeCloseTo(0.5, 6); expect(result.sparse).toBeCloseTo(0.3, 6); expect(result.dense).toBeCloseTo(0.7, 6); }); test("interpolates linearly between min and full thresholds", () => { // sparseNorm = {0.65, 1.0} → spread 0.35; midway between 0.2 and 0.5 // → factor = 0.5; effSparse = 0.5 * 0.3 = 0.15; effDense = 0.7 + 0.15. const hits = [{ sparseScore: 6.5 }, { sparseScore: 10 }]; const result = effectiveWeights(hits, 10, 0.7, 0.3, baseConfig); expect(result.spread).toBeCloseTo(0.35, 6); expect(result.sparse).toBeCloseTo(0.15, 6); expect(result.dense).toBeCloseTo(0.85, 6); }); test("config overrides min and full thresholds", () => { // Custom: min=0.0, full=1.0 — spread is now in [0, 1] linearly. const config = configWithSpreadOverrides(0.0, 1.0); const hits = [{ sparseScore: 8 }, { sparseScore: 10 }]; const result = effectiveWeights(hits, 10, 0.7, 0.3, config); // spread 0.2; factor = 0.2; effSparse = 0.06. expect(result.spread).toBeCloseTo(0.2, 6); expect(result.sparse).toBeCloseTo(0.06, 6); expect(result.dense).toBeCloseTo(0.94, 6); }); test("falls back to base weights when full <= min (degenerate config)", () => { const config = configWithSpreadOverrides(0.5, 0.3); const hits = [{ sparseScore: 5 }, { sparseScore: 10 }]; const result = effectiveWeights(hits, 10, 0.7, 0.3, config); expect(result.sparse).toBeCloseTo(0.3, 6); expect(result.dense).toBeCloseTo(0.7, 6); }); test("dense + sparse always equals baseDense + baseSparse", () => { // Property check: total weight is preserved across the spread spectrum // so `fused` stays interpretable as a [0, 1] similarity regardless of // how aggressively sparse is collapsed. const cases = [ [{ sparseScore: 1 }, { sparseScore: 1.05 }], // tiny spread [{ sparseScore: 1 }, { sparseScore: 5 }], // mid spread [{ sparseScore: 1 }, { sparseScore: 10 }], // full spread ]; for (const hits of cases) { const maxSparse = Math.max(...hits.map((h) => h.sparseScore)); const result = effectiveWeights(hits, maxSparse, 0.7, 0.3, baseConfig); expect(result.dense + result.sparse).toBeCloseTo(1.0, 6); } }); }); describe("clamp01", () => { test("passes values already in [0, 1] through unchanged", () => { expect(clamp01(0)).toBe(0); expect(clamp01(0.5)).toBe(0.5); expect(clamp01(1)).toBe(1); }); test("clamps negatives to 0 and overshoot to 1", () => { expect(clamp01(-0.5)).toBe(0); expect(clamp01(-Infinity)).toBe(0); expect(clamp01(1.0001)).toBe(1); expect(clamp01(Infinity)).toBe(1); }); }); // --------------------------------------------------------------------------- // simBatch // --------------------------------------------------------------------------- describe("simBatch", () => { test("empty candidate list returns empty map without touching backends", async () => { const config = configWithWeights(0.7, 0.3); const out = await simBatch("anything", [], config); expect(out.size).toBe(0); expect(state.embedCalls).toHaveLength(0); expect(state.sparseCalls).toHaveLength(0); expect(state.queryCalls).toHaveLength(0); }); test("empty text returns empty map without touching backends", async () => { // Turn 1 has no prior assistant message, so `computeOwnActivation` calls // `simBatch("", slugs, config)`. Gemini rejects empty content with HTTP // 400 — short-circuit here so the activation pipeline doesn't crash. const config = configWithWeights(0.7, 0.3); for (const text of ["", " ", "\n\n"]) { state.embedCalls.length = 0; state.sparseCalls.length = 0; state.queryCalls.length = 0; const out = await simBatch(text, ["alice-vscode"], config); expect(out.size).toBe(0); expect(state.embedCalls).toHaveLength(0); expect(state.sparseCalls).toHaveLength(0); expect(state.queryCalls).toHaveLength(0); } }); test("identical text yields ~1.0 when both channels max out", async () => { const config = configWithWeights(0.7, 0.3); stageHybridResponse([ // The only candidate hits both channels at their maxima // (cosine ~ 1.0; sparse score equals the batch max). { slug: "alice-vscode", denseScore: 1.0, sparseScore: 42 }, ]); const out = await simBatch( "alice prefers vs code", ["alice-vscode"], config, ); expect(out.get("alice-vscode")).toBeCloseTo(1.0, 6); }); test("orthogonal text yields ~0 when both channels miss", async () => { const config = configWithWeights(0.7, 0.3); stageHybridResponse([]); // No hits in either channel. const out = await simBatch("totally unrelated", ["alice-vscode"], config); // Slugs absent from both channels are absent from the result map. expect(out.size).toBe(0); }); test("dense-only hit gets sparse contribution of 0", async () => { const config = configWithWeights(0.7, 0.3); stageHybridResponse([ { slug: "dense-only-page", denseScore: 0.5 /* sparseScore omitted */ }, ]); const out = await simBatch("query", ["dense-only-page"], config); // cosine 0.5 (positive, passes through); 0.7 * 0.5 + 0.3 * 0 = 0.35 expect(out.get("dense-only-page")).toBeCloseTo(0.35, 6); }); test("sparse-only hit gets dense contribution of 0; sparse normalized to 1.0", async () => { const config = configWithWeights(0.7, 0.3); stageHybridResponse([ { slug: "sparse-only-page", sparseScore: 7.5 /* denseScore omitted */ }, ]); const out = await simBatch("query", ["sparse-only-page"], config); // Single entry → sparse normalizes to 1.0; 0.7 * 0 + 0.3 * 1.0 = 0.3 expect(out.get("sparse-only-page")).toBeCloseTo(0.3, 6); }); test("sparse normalization divides by per-batch max", async () => { const config = configWithWeights(0.0, 1.0); stageHybridResponse([ { slug: "alice", denseScore: 0.0, sparseScore: 10 }, { slug: "bob", denseScore: 0.0, sparseScore: 5 }, { slug: "carol", denseScore: 0.0, sparseScore: 2 }, ]); const out = await simBatch("query", ["alice", "bob", "carol"], config); // With dense_weight=0 and sparse_weight=1, scores equal the // batch-normalized sparse values: max=10 maps to 1.0, others scale. expect(out.get("alice")).toBeCloseTo(1.0, 6); expect(out.get("bob")).toBeCloseTo(0.5, 6); expect(out.get("carol")).toBeCloseTo(0.2, 6); }); test("respects the configured weight blend", async () => { const config = configWithWeights(0.4, 0.6); stageHybridResponse([ { slug: "alice", denseScore: 0.5, sparseScore: 4 }, // sparse-norm = 1.0 { slug: "bob", denseScore: 0.25, sparseScore: 2 }, // sparse-norm = 0.5 ]); const out = await simBatch("query", ["alice", "bob"], config); // Positive cosines pass through unchanged. // alice: 0.4 * 0.5 + 0.6 * 1.0 = 0.8 // bob: 0.4 * 0.25 + 0.6 * 0.5 = 0.4 expect(out.get("alice")).toBeCloseTo(0.8, 6); expect(out.get("bob")).toBeCloseTo(0.4, 6); }); test("scores are clamped into [0, 1] even when fused values overshoot", async () => { // Construct a mock response where the raw fusion exceeds 1.0; the // function still must produce <= 1.0. const config = configWithWeights(0.8, 0.5); // intentionally sums to 1.3 stageHybridResponse([ { slug: "loud-page", denseScore: 1.0, sparseScore: 1 }, // sparse-norm 1.0 ]); const out = await simBatch("query", ["loud-page"], config); expect(out.get("loud-page")).toBe(1); }); test("forwards the candidate slugs as a Qdrant slug-IN filter on every channel", async () => { const config = configWithWeights(0.7, 0.3); stageHybridResponse([]); await simBatch("query", ["alice", "bob", "carol"], config); // All four channels (body dense + sparse, summary dense + sparse) ran // with the same slug-restriction filter and the same per-channel limit // equal to the candidate count. expect(state.queryCalls).toHaveLength(4); for (const call of state.queryCalls) { expect(call.limit).toBe(3); expect(call.filter).toEqual({ must: [{ key: "slug", match: { any: ["alice", "bob", "carol"] } }], }); } }); test("committed-dimension/reachable-backend mismatch skips the dense embed and degrades to sparse-only", async () => { // Simulates a 3072-dim collection committed while only a 384-dim backend // is reachable: the dense embed is skipped (no `embedWithBackend` throw), // and `hybridQueryConceptPages` runs sparse-only via the empty dense vector // so BM25 still scores the candidate. state.dimensionAvailable = false; const config = configWithWeights(0.0, 1.0); stageHybridResponse([ { slug: "sparse-only-page", sparseScore: 7.5 /* denseScore omitted */ }, ]); const out = await simBatch("query", ["sparse-only-page"], config); // No dense embed round-trip was paid. expect(state.embedCalls).toHaveLength(0); // Sparse channels still ran; the dense channels were skipped (empty dense // vector), so only the two sparse `using` channels hit Qdrant. expect(state.queryCalls.map((c) => c.using).sort()).toEqual([ "sparse", "summary_sparse", ]); // BM25 still produced a score for the candidate: sparse normalized to 1.0 // (single hit), sparse_weight 1.0 → fused 1.0. expect(out.get("sparse-only-page")).toBeCloseTo(1.0, 6); }); test("embeds the query text exactly once via dense + sparse backends", async () => { const config = configWithWeights(0.7, 0.3); stageHybridResponse([]); await simBatch("hello world", ["alice"], config); expect(state.embedCalls).toHaveLength(1); expect(state.embedCalls[0].inputs).toEqual(["hello world"]); expect(state.sparseCalls).toEqual(["hello world"]); }); test("takes max(body, summary) per slug — summary higher than body wins", async () => { // Body channels return a modest score; summary channels return a much // higher score. The max collapses to the summary score. const config = configWithWeights(1.0, 0.0); stageHybridResponse([ { slug: "alice", denseScore: 0.3, summaryDenseScore: 0.7, }, ]); const out = await simBatch("query", ["alice"], config); // Positive cosines pass through unchanged: max(0.3, 0.7) = 0.7. expect(out.get("alice")).toBeCloseTo(0.7, 6); }); test("takes max(body, summary) per slug — body higher than summary wins", async () => { // Inverse case: body dominates, max stays at body. const config = configWithWeights(1.0, 0.0); stageHybridResponse([ { slug: "alice", denseScore: 0.9, summaryDenseScore: 0.4, }, ]); const out = await simBatch("query", ["alice"], config); // Positive cosines pass through unchanged: max(0.9, 0.4) = 0.9. expect(out.get("alice")).toBeCloseTo(0.9, 6); }); test("falls back to body-only when the page has no summary embedding", async () => { // Pages predating the summary field have no summary_dense/sparse vectors. // Their summary channels return no hits — the max collapses to body. const config = configWithWeights(1.0, 0.0); stageHybridResponse([ { slug: "legacy-page", denseScore: 0.6, // summaryDenseScore / summarySparseScore omitted }, ]); const out = await simBatch("query", ["legacy-page"], config); // Positive cosine 0.6 passes through unchanged. expect(out.get("legacy-page")).toBeCloseTo(0.6, 6); }); test("normalizes body and summary sparse channels independently", async () => { // Summary sparse scores live on a different scale than body sparse — // a small absolute summary-sparse value (1.5) on the only page that // has summary signal still normalizes to 1.0 within the summary // channel, so the summary-only fused score should win out. const config = configWithWeights(0.0, 1.0); stageHybridResponse([ { slug: "alice", denseScore: 0.0, sparseScore: 100, // body sparse max in this batch }, { slug: "bob", denseScore: 0.0, sparseScore: 0.5, // body sparse normalized = 0.005 summaryDenseScore: 0.0, summarySparseScore: 1.5, // summary sparse max in this batch }, ]); const out = await simBatch("query", ["alice", "bob"], config); // Alice has only body. Body sparse normalized to 1.0; sparse_weight=1.0 → 1.0. expect(out.get("alice")).toBeCloseTo(1.0, 6); // Bob's summary side normalizes its 1.5 (only sparse-bearing summary // hit) — a single sparse-bearing hit is below the adaptive-spread // floor, so the channel collapses to base weights and the lone // sparseNormalized=1.0 hit yields a fused summary score of 1.0. // Body side has only bob's tiny sparse=0.5 against the body batch max // of 100 → ~0.005. The max picks the summary side. expect(out.get("bob")).toBeCloseTo(1.0, 6); }); test("dense-favored config keeps the dense-strong candidate above the sparse-strong one", async () => { // Regression guard: an earlier fix mapped cosines into [0, 1] via // `(cos + 1) / 2` before fusion, which halved every pairwise dense // difference and silently shifted ranking toward sparse. With dense // weighted higher than sparse, the dense-only hit must outrank the // sparse-only hit even though sparse normalizes to 1.0. const config = configWithWeights(0.7, 0.3); stageHybridResponse([ { slug: "dense-strong", denseScore: 0.5 /* sparseScore omitted */ }, { slug: "sparse-strong", denseScore: 0.0, sparseScore: 1 }, ]); const out = await simBatch( "query", ["dense-strong", "sparse-strong"], config, ); // dense-strong: 0.7 * max(0, 0.5) + 0.3 * 0 = 0.35 // sparse-strong: 0.7 * max(0, 0.0) + 0.3 * 1.0 = 0.30 expect(out.get("dense-strong")).toBeCloseTo(0.35, 6); expect(out.get("sparse-strong")).toBeCloseTo(0.3, 6); expect(out.get("dense-strong")!).toBeGreaterThan(out.get("sparse-strong")!); }); test("negative cosine maps to a non-negative dense contribution", async () => { // Qdrant cosine search returns scores in [-1, 1]. A near-zero or // negative cosine must not yield a negative dense contribution that // depresses the fused score below the sparse-only floor. const config = configWithWeights(0.7, 0.3); stageHybridResponse([ { slug: "anti-match", denseScore: -1.0, sparseScore: 1 }, ]); const out = await simBatch("query", ["anti-match"], config); // cosine -1 → clamped to 0; sparse-norm = 1.0; fused = 0.7*0 + 0.3*1 = 0.3. expect(out.get("anti-match")).toBeCloseTo(0.3, 6); }); test("returned scores are always in [0, 1] for arbitrary inputs", async () => { const config = configWithWeights(0.7, 0.3); stageHybridResponse([ { slug: "a", denseScore: 0.99, sparseScore: 100 }, { slug: "b", denseScore: 0.5, sparseScore: 50 }, { slug: "c", denseScore: 0.0, sparseScore: 1 }, { slug: "d", denseScore: 0.123, sparseScore: 0 }, // explicit zero ]); const out = await simBatch("query", ["a", "b", "c", "d"], config); for (const [, score] of out) { expect(score).toBeGreaterThanOrEqual(0); expect(score).toBeLessThanOrEqual(1); } }); });