/** * Hybrid search orchestrator. * Combines BM25, vector search, expansion, fusion, and reranking. * * @module src/pipeline/hybrid */ import type { Config } from "../config/types"; import type { EmbeddingPort, GenerationPort, RerankPort } from "../llm/types"; import type { DocumentRow, StorePort } from "../store/types"; import type { VectorIndexPort, VectorSearchOptions, } from "../store/vector/types"; import type { ExpansionResult, ExplainLine, HybridSearchOptions, PipelineConfig, QueryDiagnoseTrace, QueryDiagnoseTraceCandidate, SearchResult, SearchResults, } from "./types"; import { normalizeContentTypes } from "../config/content-types"; import { projectRecordEvidenceMetadata } from "../core/record-metadata"; import { normalizeMetadataPredicate } from "../core/typed-metadata"; import { embedTextsWithRecovery } from "../embed/batch"; import { assertInferenceActive, assertInferenceResult, withInferenceScope, } from "../llm/inference-scope"; import { err, ok } from "../store/types"; import { resolveVectorSearchIdentity } from "../store/vector/variant-search"; import { createChunkLookup } from "./chunk-lookup"; import { attachAuxiliaryScoreMetadata, hasAuxiliaryRanking, scoreContentTypeBoost, sortByFinalScoreStable, } from "./content-type-boost"; import { formatQueryForEmbedding } from "./contextual"; import { attachSearchResultEgressLineage } from "./egress-lineage"; import { expandQuery } from "./expansion"; import { buildExplainResults, type ExpansionStatus, explainBm25, explainCounters, explainExpansion, explainFusion, explainQueryModes, explainRerank, explainTimings, explainVector, } from "./explain"; import { evaluateDocumentChunkFilters, typedMetadataWarnings } from "./filters"; import { type RankedInput, rrfFuse, toRankedInput } from "./fusion"; import { expandGraphCandidates } from "./graph-retrieval"; import { RequestHydration } from "./hydration"; import { selectBestChunkForSteering } from "./intent"; import { OwnerMetadataError, resolveFusionOwners } from "./owner-fusion"; import { hasProjectAffinity } from "./project-affinity"; import { detectQueryLanguage } from "./query-language"; import { buildExpansionFromQueryModes, summarizeQueryModes, } from "./query-modes"; import { rerankCandidates } from "./rerank"; import { attachSearchResultContexts } from "./result-context"; import { cleanDisplaySnippet } from "./snippet"; import { isWithinTemporalRange, resolveRecencyTimestamp, resolveTemporalRange, shouldSortByRecency, } from "./temporal"; import { attachSearchResultPlannerMetadata, attachSearchResultsTraceMetadata, } from "./trace-metadata"; import { DEFAULT_PIPELINE_CONFIG, SEARCH_RESULT_PLANNER_METADATA, } from "./types"; // ───────────────────────────────────────────────────────────────────────────── // Dependencies // ───────────────────────────────────────────────────────────────────────────── export interface HybridSearchDeps { store: StorePort; config: Config; vectorIndex: VectorIndexPort | null; embedPort: EmbeddingPort | null; expandPort: GenerationPort | null; rerankPort: RerankPort | null; pipelineConfig?: PipelineConfig; /** Internal request owner; the creating caller releases it. */ hydration?: RequestHydration; } // ───────────────────────────────────────────────────────────────────────────── // Score Normalization // ───────────────────────────────────────────────────────────────────────────── // Removed: _normalizeVectorScore was dead code (vector distances normalized in vector index) // ───────────────────────────────────────────────────────────────────────────── // BM25 Score Normalization // ───────────────────────────────────────────────────────────────────────────── /** * Normalize raw BM25 score to 0-1 range using sigmoid. * BM25 scores are negative in SQLite FTS5 (more negative = better match). * Typical range: -15 (excellent) to -2 (weak match). * Maps to 0-1 where higher is better. */ function normalizeBm25Score(rawScore: number): number { const absScore = Math.abs(rawScore); // Sigmoid with center=4.5, scale=2.8 // Maps: -15 → ~0.99, -5 → ~0.55, -2 → ~0.29 return 1 / (1 + Math.exp(-(absScore - 4.5) / 2.8)); } // ───────────────────────────────────────────────────────────────────────────── // BM25 Strength Check // ───────────────────────────────────────────────────────────────────────────── // Thresholds for strong signal detection (conservative - prefer expansion over speed) const STRONG_TOP_SCORE = 0.84; // ~84th percentile confidence const STRONG_GAP = 0.14; // Clear separation from #2 /** * Check if BM25 results are strong enough to skip expansion. * Returns true if top result is both confident AND clearly separated. * This prevents skipping on weak-but-separated results. */ async function checkBm25Strength( store: StorePort, query: string, options?: { collection?: string; lang?: string; tagsAll?: string[]; tagsAny?: string[]; since?: string; until?: string; categories?: string[]; author?: string; filter?: HybridSearchOptions["filter"]; relPathPrefix?: string; allowedMirrorHashes?: string[]; exclude?: string[]; memoryScopesAny?: string[]; excludeSuperseded?: boolean; } ): Promise { const result = await store.searchFts(query, { limit: 5, collection: options?.collection, relPathPrefix: options?.relPathPrefix, language: options?.lang, chunkLanguage: options?.lang, allowedMirrorHashes: options?.allowedMirrorHashes, exclude: options?.exclude, excludeMetadata: true, semanticMetadata: true, memoryScopesAny: options?.memoryScopesAny, excludeSuperseded: options?.excludeSuperseded, tagsAll: options?.tagsAll, tagsAny: options?.tagsAny, since: options?.since, until: options?.until, categories: options?.categories, author: options?.author, filter: options?.filter, }); if (!result.ok || result.value.length === 0) { return false; } // Normalize scores (higher = better) const scores = result.value .map((r) => normalizeBm25Score(r.score)) .sort((a, b) => b - a); // Descending const topScore = scores[0] ?? 0; const secondScore = scores[1] ?? 0; const gap = topScore - secondScore; // Strong signal requires BOTH: high confidence AND clear separation return topScore >= STRONG_TOP_SCORE && gap >= STRONG_GAP; } // ───────────────────────────────────────────────────────────────────────────── // FTS Retrieval (returns ChunkIds) // ───────────────────────────────────────────────────────────────────────────── interface ChunkId { sourceDocid?: string; documentIds?: number[]; mirrorHash: string; seq: number; score?: number; } type FtsChunksResult = | { ok: true; chunks: ChunkId[] } | { ok: false; code: "INVALID_INPUT" | "OTHER"; message: string }; async function searchFtsChunks( store: StorePort, query: string, options: { limit: number; collection?: string; lang?: string; tagsAll?: string[]; tagsAny?: string[]; since?: string; until?: string; categories?: string[]; author?: string; filter?: HybridSearchOptions["filter"]; relPathPrefix?: string; allowedMirrorHashes?: string[]; exclude?: string[]; memoryScopesAny?: string[]; excludeSuperseded?: boolean; } ): Promise { const result = await store.searchFts(query, { limit: options.limit, collection: options.collection, relPathPrefix: options.relPathPrefix, language: options.lang, chunkLanguage: options.lang, excludeMetadata: true, semanticMetadata: true, memoryScopesAny: options.memoryScopesAny, excludeSuperseded: options.excludeSuperseded, exclude: options.exclude, allowedMirrorHashes: options.allowedMirrorHashes, tagsAll: options.tagsAll, tagsAny: options.tagsAny, since: options.since, until: options.until, categories: options.categories, author: options.author, filter: options.filter, }); if (!result.ok) { // Propagate INVALID_INPUT for FTS syntax errors const code = result.error.code === "INVALID_INPUT" ? "INVALID_INPUT" : "OTHER"; return { ok: false, code, message: result.error.message }; } return { ok: true, chunks: result.value.map((r) => ({ sourceDocid: r.docid, mirrorHash: r.mirrorHash, seq: r.seq, score: r.score, })), }; } // ───────────────────────────────────────────────────────────────────────────── // Vector Retrieval (returns ChunkIds) // ───────────────────────────────────────────────────────────────────────────── async function searchVectorChunks( vectorIndex: VectorIndexPort, embedPort: EmbeddingPort, query: string, options: { limit: number; minScore?: number; allowedMirrorHashes?: string[]; eligibility?: VectorSearchOptions["eligibility"]; } ): Promise<{ ok: true; chunks: ChunkId[] } | { ok: false; reason: string }> { if (!vectorIndex.searchAvailable) { return { ok: false, reason: "vector_unavailable" }; } // Embed query with contextual formatting const embedResult = await embedPort.embed( formatQueryForEmbedding(query, embedPort.modelUri) ); assertInferenceResult(embedResult); if (!embedResult.ok) { return { ok: false, reason: "vector_embed_error" }; } const queryEmbedding = new Float32Array(embedResult.value); const searchResult = await vectorIndex.searchNearest( queryEmbedding, options.limit, { embeddingIdentity: resolveVectorSearchIdentity(embedPort), minScore: options.minScore, allowedMirrorHashes: options.allowedMirrorHashes, eligibility: options.eligibility, } ); if (!searchResult.ok) { return { ok: false, reason: "vector_search_error" }; } return { ok: true, chunks: searchResult.value.map((r) => ({ documentIds: r.documentIds, mirrorHash: r.mirrorHash, seq: r.seq, score: r.distance, })), }; } function toTraceCandidates(chunks: ChunkId[]): QueryDiagnoseTraceCandidate[] { return chunks.map((chunk, index) => ({ mirrorHash: chunk.mirrorHash, seq: chunk.seq, rank: index + 1, score: chunk.score ?? index + 1, })); } function candidatesToTrace( candidates: Array<{ mirrorHash: string; seq: number; fusionScore?: number; blendedScore?: number; }> ): QueryDiagnoseTraceCandidate[] { return candidates.map((candidate, index) => ({ mirrorHash: candidate.mirrorHash, seq: candidate.seq, rank: index + 1, score: candidate.blendedScore ?? candidate.fusionScore ?? index + 1, })); } // ───────────────────────────────────────────────────────────────────────────── // Hybrid Search // ───────────────────────────────────────────────────────────────────────────── /** * Execute hybrid search with full pipeline. */ export async function searchHybrid( deps: HybridSearchDeps, query: string, options: HybridSearchOptions = {} ): Promise>> { if (options.filter !== undefined) options = { ...options, filter: normalizeMetadataPredicate(options.filter), }; const hydration = deps.hydration ?? new RequestHydration(deps.store); try { return await withInferenceScope(options, () => searchHybridWithHydration(deps, query, options, hydration) ); } catch (cause) { if (cause instanceof OwnerMetadataError) return err("QUERY_FAILED", cause.message); throw cause; } finally { if (!deps.hydration) hydration.release(); } } // oxlint-disable-next-line max-lines-per-function -- search orchestration with BM25, vector, fusion, reranking async function searchHybridWithHydration( deps: HybridSearchDeps, query: string, options: HybridSearchOptions, hydration: RequestHydration ): Promise>> { const runStartedAt = performance.now(); const { store, vectorIndex, embedPort, expandPort, rerankPort } = deps; const pipelineConfig = deps.pipelineConfig ?? DEFAULT_PIPELINE_CONFIG; const contentTypeRules = options.contentTypeRules ?? normalizeContentTypes(deps.config.contentTypes ?? []).rules; const auxiliaryRankingActive = hasAuxiliaryRanking( options.projectAffinity, contentTypeRules ); const limit = options.limit ?? 20; const recencySort = shouldSortByRecency(query); const temporalRange = resolveTemporalRange( query, options.since, options.until ); const explainLines: ExplainLine[] = []; let expansion: ExpansionResult | null = null; const timings = { langMs: 0, expansionMs: 0, bm25Ms: 0, vectorMs: 0, graphMs: 0, fusionMs: 0, rerankMs: 0, assemblyMs: 0, totalMs: 0, }; const counters = { expansionCacheHits: 0, expansionCacheLookups: 0, rerankCacheHits: 0, rerankCacheLookups: 0, fallbackEvents: [] as string[], }; // Increase retrieval limits when post-retrieval filters are active. const hasPostFilters = Boolean( options.tagsAll?.length || options.tagsAny?.length || options.categories?.length || options.author || temporalRange.since || temporalRange.until ); const retrievalMultiplier = hasPostFilters || recencySort ? 3 : 1; // ───────────────────────────────────────────────────────────────────────── // 0. Detect query language for PROMPT SELECTION only // CRITICAL: Detection does NOT change retrieval filters - options.lang does // Priority: queryLanguageHint (MCP) > lang (CLI) > detection // ───────────────────────────────────────────────────────────────────────── const langStartedAt = performance.now(); const detection = detectQueryLanguage(query); // Use explicit hint > lang filter > detected language const queryLanguage = options.queryLanguageHint ?? options.lang ?? detection.bcp47; // Build explain message for language detection let langMessage: string; if (options.queryLanguageHint) { langMessage = `queryLanguage=${queryLanguage} (hint)`; } else if (options.lang) { langMessage = `queryLanguage=${queryLanguage} (explicit)`; } else { const confidence = detection.confident ? "" : ", low confidence"; langMessage = `queryLanguage=${queryLanguage} (detected${confidence})`; } explainLines.push({ stage: "lang", message: langMessage }); timings.langMs = performance.now() - langStartedAt; // ───────────────────────────────────────────────────────────────────────── // 1. Check if expansion needed // ───────────────────────────────────────────────────────────────────────── const expansionStartedAt = performance.now(); const shouldExpand = !options.noExpand && expandPort !== null; let expansionStatus: ExpansionStatus = "disabled"; let queryModeSummary: ReturnType | undefined = undefined; if (options.queryModes?.length) { queryModeSummary = summarizeQueryModes(options.queryModes); explainLines.push(explainQueryModes(queryModeSummary)); expansion = buildExpansionFromQueryModes(options.queryModes); expansionStatus = "provided"; } if (expansionStatus !== "provided" && shouldExpand) { const hasStrongSignal = options.intent?.trim() ? false : await checkBm25Strength(store, query, { collection: options.collection, lang: options.lang, memoryScopesAny: options.memoryFilter?.scopes, excludeSuperseded: options.memoryFilter?.excludeSuperseded, exclude: options.exclude, allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes, tagsAll: options.tagsAll, tagsAny: options.tagsAny, since: temporalRange.since, until: temporalRange.until, categories: options.categories, author: options.author, filter: options.filter, relPathPrefix: options.retrievalScope?.relPathPrefix, }); if (hasStrongSignal) { expansionStatus = "skipped_strong"; counters.fallbackEvents.push("expansion_skipped_strong"); } else { expansionStatus = "attempted"; const expandResult = await expandQuery(expandPort, query, { // Use queryLanguage for prompt selection, NOT options.lang (retrieval filter) lang: queryLanguage, timeout: pipelineConfig.expansionTimeout, intent: options.intent, contextSize: deps.config.models?.expandContextSize, }); if (expandResult.ok) { expansion = expandResult.value; } else { counters.fallbackEvents.push("expansion_error"); } } } if (expansionStatus === "disabled") { counters.fallbackEvents.push("expansion_disabled"); } explainLines.push(explainExpansion(expansionStatus, expansion)); timings.expansionMs = performance.now() - expansionStartedAt; // ───────────────────────────────────────────────────────────────────────── // 2. Parallel retrieval using raw store/vector APIs for correct seq tracking // ───────────────────────────────────────────────────────────────────────── const rankedInputs: RankedInput[] = []; const diagnoseTrace: QueryDiagnoseTrace | undefined = options.diagnoseTrace ? { stages: [] } : undefined; const vectorEligibility: VectorSearchOptions["eligibility"] = { collection: options.collection, memoryScopesAny: options.memoryFilter?.scopes, excludeSuperseded: options.memoryFilter?.excludeSuperseded, relPathPrefix: options.retrievalScope?.relPathPrefix, tagsAll: options.tagsAll, tagsAny: options.tagsAny, since: temporalRange.since, until: temporalRange.until, categories: options.categories, author: options.author, filter: options.filter, exclude: options.exclude, excludeMetadata: true, semanticMetadata: true, language: options.lang, }; const bm25StartedAt = performance.now(); // BM25: original query const bm25Result = await searchFtsChunks(store, query, { limit: limit * 2 * retrievalMultiplier, memoryScopesAny: options.memoryFilter?.scopes, excludeSuperseded: options.memoryFilter?.excludeSuperseded, exclude: options.exclude, allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes, collection: options.collection, lang: options.lang, tagsAll: options.tagsAll, tagsAny: options.tagsAny, since: temporalRange.since, until: temporalRange.until, categories: options.categories, author: options.author, filter: options.filter, relPathPrefix: options.retrievalScope?.relPathPrefix, }); // Propagate FTS syntax errors as INVALID_INPUT if (!bm25Result.ok && bm25Result.code === "INVALID_INPUT") { return err("INVALID_INPUT", `Invalid search query: ${bm25Result.message}`); } // Other errors: continue with empty BM25 results const bm25Chunks = bm25Result.ok ? bm25Result.chunks : []; const bm25Count = bm25Chunks.length; diagnoseTrace?.stages.push({ id: "bm25", status: "active", sourceCount: 1, candidates: toTraceCandidates(bm25Chunks), }); if (bm25Count > 0) { rankedInputs.push(toRankedInput("bm25", bm25Chunks)); } // BM25: lexical variants (optional; run in parallel and ignore failures) if (expansion?.lexicalQueries?.length) { const lexicalVariantResults = await Promise.allSettled( expansion.lexicalQueries.map((variant) => searchFtsChunks(store, variant, { limit: limit * retrievalMultiplier, memoryScopesAny: options.memoryFilter?.scopes, excludeSuperseded: options.memoryFilter?.excludeSuperseded, exclude: options.exclude, allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes, collection: options.collection, lang: options.lang, tagsAll: options.tagsAll, tagsAny: options.tagsAny, since: temporalRange.since, until: temporalRange.until, categories: options.categories, author: options.author, filter: options.filter, relPathPrefix: options.retrievalScope?.relPathPrefix, }) ) ); for (const settled of lexicalVariantResults) { assertInferenceActive(); if (settled.status !== "fulfilled") { continue; } const variantResult = settled.value; if (variantResult.ok && variantResult.chunks.length > 0) { rankedInputs.push(toRankedInput("bm25_variant", variantResult.chunks)); } } } timings.bm25Ms = performance.now() - bm25StartedAt; explainLines.push(explainBm25(bm25Count)); // Vector search let vecCount = 0; let vectorsUsed = false; const vectorAvailable = (vectorIndex?.searchAvailable && embedPort !== null) ?? false; if (!vectorAvailable) { counters.fallbackEvents.push("vector_unavailable"); } const vectorStartedAt = performance.now(); const vectorTraceChunks: ChunkId[] = []; if (vectorAvailable && vectorIndex && embedPort) { const vectorVariantQueries = [ ...(expansion?.vectorQueries?.map((query) => ({ source: "vector_variant" as const, query, })) ?? []), ...(expansion?.hyde ? [{ source: "hyde" as const, query: expansion.hyde }] : []), ]; if (vectorVariantQueries.length === 0) { const vectorResult = await searchVectorChunks( vectorIndex, embedPort, query, { limit: limit * 2 * retrievalMultiplier, allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes, eligibility: vectorEligibility, } ); if (!vectorResult.ok) counters.fallbackEvents.push(vectorResult.reason); else vectorsUsed = true; const vecChunks = vectorResult.ok ? vectorResult.chunks : []; vecCount = vecChunks.length; vectorTraceChunks.push(...vecChunks); if (vecCount > 0) { rankedInputs.push(toRankedInput("vector", vecChunks)); } } else { const batchedQueries = [ { source: "vector" as const, query, limit: limit * 2 * retrievalMultiplier, }, ...vectorVariantQueries.map((variant) => ({ ...variant, limit: limit * retrievalMultiplier, })), ]; const embedResult = await embedTextsWithRecovery( embedPort, batchedQueries.map((variant) => formatQueryForEmbedding(variant.query, embedPort.modelUri) ) ); assertInferenceResult(embedResult); if (!embedResult.ok) { counters.fallbackEvents.push("vector_embed_error"); } else { if (embedResult.value.batchFailed) { counters.fallbackEvents.push("vector_embed_batch_fallback"); } for (const [index, variant] of batchedQueries.entries()) { assertInferenceActive(); const embedding = embedResult.value.vectors[index]; if (!embedding || !variant) { continue; } const searchResult = await vectorIndex.searchNearest( new Float32Array(embedding), variant.limit, { embeddingIdentity: resolveVectorSearchIdentity(embedPort), allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes, eligibility: vectorEligibility, } ); if (!searchResult.ok) { counters.fallbackEvents.push("vector_search_error"); continue; } vectorsUsed = true; if (searchResult.value.length === 0) continue; const chunks = searchResult.value.map((item) => ({ documentIds: item.documentIds, mirrorHash: item.mirrorHash, seq: item.seq, })); if (variant.source === "vector") { vecCount = chunks.length; } vectorTraceChunks.push(...chunks); if (chunks.length === 0) { continue; } rankedInputs.push(toRankedInput(variant.source, chunks)); } } } } timings.vectorMs = performance.now() - vectorStartedAt; diagnoseTrace?.stages.push({ id: "vector", status: vectorAvailable ? "active" : "skipped", reason: vectorAvailable ? undefined : "vector_unavailable", sourceCount: vectorAvailable ? 1 : 0, candidates: toTraceCandidates(vectorTraceChunks), }); explainLines.push( explainVector( vecCount, vectorAvailable, vectorIndex?.loadError, vectorIndex?.guidance ) ); // ───────────────────────────────────────────────────────────────────────── // 3. RRF Fusion // ───────────────────────────────────────────────────────────────────────── const fusionStartedAt = performance.now(); const candidateLimit = options.candidateLimit ?? pipelineConfig.rerankCandidates; let fusedCandidates = rrfFuse( await resolveFusionOwners(rankedInputs, store, hydration, query, options), pipelineConfig.rrf ); diagnoseTrace?.stages.push({ id: "fusion", status: "active", sourceCount: rankedInputs.length, candidates: candidatesToTrace(fusedCandidates), }); timings.fusionMs = performance.now() - fusionStartedAt; const graphStartedAt = performance.now(); const graphExpansion = await expandGraphCandidates( store, fusedCandidates, { collection: options.collection, includeSimilar: vectorAvailable, eligibility: vectorEligibility, limit, candidateLimit, disabled: options.graph === false || options.noGraph === true, relPathPrefix: options.retrievalScope?.relPathPrefix, lang: options.lang, tagsAll: options.tagsAll, tagsAny: options.tagsAny, since: temporalRange.since, until: temporalRange.until, categories: options.categories, author: options.author, filter: options.filter, }, hydration ); timings.graphMs = performance.now() - graphStartedAt; if (graphExpansion.candidates.length > 0) { const graphFusionStartedAt = performance.now(); rankedInputs.push(toRankedInput("graph", graphExpansion.candidates)); fusedCandidates = rrfFuse( await resolveFusionOwners(rankedInputs, store, hydration, query, options), pipelineConfig.rrf ); timings.fusionMs += performance.now() - graphFusionStartedAt; } diagnoseTrace?.stages.push({ id: "graph", status: graphExpansion.meta.attempted ? "active" : "skipped", reason: graphExpansion.meta.attempted ? undefined : graphExpansion.meta.fallbackReasons.join(", ") || "disabled", sourceCount: graphExpansion.meta.attempted ? 1 : 0, candidates: graphExpansion.candidates.map((candidate, index) => ({ mirrorHash: candidate.mirrorHash, seq: candidate.seq, rank: index + 1, score: index + 1, })), }); if (graphExpansion.meta.fallbackReasons.length > 0) { counters.fallbackEvents.push(...graphExpansion.meta.fallbackReasons); } explainLines.push({ stage: "graph", message: graphExpansion.meta.attempted ? `seeds=${graphExpansion.meta.seedCount}, candidates=${graphExpansion.meta.candidateCount}/${graphExpansion.meta.maxCandidates}, explicit=${graphExpansion.meta.edgeConfidence.explicit}, inferred=${graphExpansion.meta.edgeConfidence.inferred}, ambiguous=${graphExpansion.meta.edgeConfidence.ambiguous}, similarity=${graphExpansion.meta.edgeConfidence.similarity}` : `skipped (${graphExpansion.meta.fallbackReasons.join(", ") || "disabled"})`, }); explainLines.push( explainFusion(pipelineConfig.rrf.k, fusedCandidates.length) ); // Auxiliary scores enter after fusion normalization and before rerank // blending. The reranker therefore remains the final ordering authority, // including its lexical top-hit guardrail. let prefetchedDocuments: DocumentRow[] | undefined; const auxiliaryBaseScores = new Map(); const preRerankAdjustedCandidates = new Set(); let adjustNormalizedFusionScore: | (( candidate: (typeof fusedCandidates)[number], normalizedScore: number ) => number) | undefined; if (auxiliaryRankingActive) { const prefetchedDocumentsResult = await hydration.getDocumentsByMirrorHashes( [...new Set(fusedCandidates.map((candidate) => candidate.mirrorHash))], { collection: options.collection, activeOnly: true, } ); if (!prefetchedDocumentsResult.ok) { return err("QUERY_FAILED", prefetchedDocumentsResult.error.message); } prefetchedDocuments = prefetchedDocumentsResult.value; const scoringDocumentsByHash = new Map(); for (const document of [...prefetchedDocuments].sort((left, right) => { if (left.uri !== right.uri) return left.uri.localeCompare(right.uri); return left.docid.localeCompare(right.docid); })) { if (!document.mirrorHash) continue; const documents = scoringDocumentsByHash.get(document.mirrorHash) ?? []; documents.push(document); scoringDocumentsByHash.set(document.mirrorHash, documents); } adjustNormalizedFusionScore = (candidate, normalizedScore) => { const candidateKey = `${candidate.mirrorHash}:${candidate.seq}:${candidate.documentId ?? ""}`; auxiliaryBaseScores.set(candidateKey, normalizedScore); const documents = scoringDocumentsByHash .get(candidate.mirrorHash) ?.filter( (doc) => candidate.documentId === undefined || candidate.documentId === doc.id ); if (!documents?.length) return normalizedScore; const projectedScores = documents.map( (document) => scoreContentTypeBoost( normalizedScore, document.contentType ?? undefined, document.contentTypeSource, document.relPath, document.collection, contentTypeRules, options.projectAffinity, { kind: "hybrid_blended", score: normalizedScore } ).projectAffinity.finalScore ); const agreedScore = projectedScores[0] ?? normalizedScore; const projectionsAgree = projectedScores.every( (score) => Math.abs(score - agreedScore) < 1e-9 ); if (!projectionsAgree) return normalizedScore; preRerankAdjustedCandidates.add(candidateKey); return agreedScore; }; } // ───────────────────────────────────────────────────────────────────────── // 4. Reranking // ───────────────────────────────────────────────────────────────────────── const rerankStartedAt = performance.now(); const rerankResult = await rerankCandidates( { rerankPort: options.noRerank ? null : rerankPort, store, hydration }, query, fusedCandidates, { maxCandidates: candidateLimit, blendingSchedule: pipelineConfig.blendingSchedule, intent: options.intent, adjustNormalizedFusionScore, } ); if (rerankResult.fallbackReason === "disabled") { counters.fallbackEvents.push("rerank_disabled"); } else if (rerankResult.fallbackReason === "error") { counters.fallbackEvents.push("rerank_error"); } timings.rerankMs = performance.now() - rerankStartedAt; diagnoseTrace?.stages.push({ id: "rerank", status: rerankResult.reranked ? "active" : "skipped", reason: rerankResult.reranked ? undefined : (rerankResult.fallbackReason ?? "disabled"), sourceCount: rerankResult.reranked ? 1 : 0, candidates: candidatesToTrace(rerankResult.candidates), }); explainLines.push( explainRerank(!options.noRerank && rerankPort !== null, candidateLimit) ); // ───────────────────────────────────────────────────────────────────────── // 4b. Apply minScore filter (blendedScore is now normalized to [0,1]) // ───────────────────────────────────────────────────────────────────────── const minScore = options.minScore ?? 0; const filteredCandidates = minScore > 0 && !auxiliaryRankingActive ? rerankResult.candidates.filter((c) => c.blendedScore >= minScore) : rerankResult.candidates; // ───────────────────────────────────────────────────────────────────────── // 5. Build final results (optimized: batch lookups, no per-candidate queries) // ───────────────────────────────────────────────────────────────────────── const assemblyStartedAt = performance.now(); // Collect unique mirrorHashes needed from candidates. const neededHashes = new Set(filteredCandidates.map((c) => c.mirrorHash)); // Fetch only needed documents and collections. let documents = prefetchedDocuments; if (!documents) { const docsResult = await hydration.getDocumentsByMirrorHashes( [...neededHashes], { collection: options.collection, activeOnly: true, } ); if (!docsResult.ok) { return err("QUERY_FAILED", docsResult.error.message); } documents = docsResult.value; } const collectionsResult = await store.getCollections(); // Build lookup maps. const docsByMirrorHash = new Map(); const addDocument = (doc: DocumentRow): void => { if (!doc.mirrorHash) return; const docs = docsByMirrorHash.get(doc.mirrorHash) ?? []; docs.push(doc); docsByMirrorHash.set(doc.mirrorHash, docs); }; const matchesMetadataFilters = (doc: DocumentRow): boolean => { const relPathPrefix = options.retrievalScope?.relPathPrefix; const sourceRelPath = doc.recordSourcePath ?? doc.relPath; if ( relPathPrefix !== undefined && sourceRelPath !== relPathPrefix && !sourceRelPath.startsWith(`${relPathPrefix}/`) ) { return false; } if (!isWithinTemporalRange(doc.sourceMtime, temporalRange)) { return false; } if ( options.author && !doc.author?.toLowerCase().includes(options.author.toLowerCase()) ) { return false; } if (options.categories?.length) { const allowed = new Set(options.categories.map((c) => c.toLowerCase())); const contentTypeMatch = doc.contentType ? allowed.has(doc.contentType.toLowerCase()) : false; const categoryMatch = (doc.categories ?? []).some((c) => allowed.has(c.toLowerCase()) ); if (!contentTypeMatch && !categoryMatch) { return false; } } return true; }; // Collect doc IDs that need tag filtering const needsTagFilter = options.tagsAll?.length || options.tagsAny?.length; const docIdsForTagCheck: number[] = []; const candidateDocs: DocumentRow[] = []; for (const doc of documents) { assertInferenceActive(); if (!doc.mirrorHash) { continue; } if (needsTagFilter) { docIdsForTagCheck.push(doc.id); candidateDocs.push(doc); } else { if (matchesMetadataFilters(doc)) { addDocument(doc); } } } // Apply tag filters if needed (batch fetch to avoid N+1) if (needsTagFilter && docIdsForTagCheck.length > 0) { const tagsResult = await store.getTagsBatch(docIdsForTagCheck); if (tagsResult.ok) { const tagsByDocId = tagsResult.value; for (const doc of candidateDocs) { assertInferenceActive(); const docTags = new Set( (tagsByDocId.get(doc.id) ?? []).map((t) => t.tag) ); // tagsAll: doc must have ALL specified tags if (options.tagsAll?.length) { const hasAll = options.tagsAll.every((t) => docTags.has(t)); if (!hasAll) continue; } // tagsAny: doc must have at least one of the specified tags if (options.tagsAny?.length) { const hasAny = options.tagsAny.some((t) => docTags.has(t)); if (!hasAny) continue; } if (doc.mirrorHash && matchesMetadataFilters(doc)) { addDocument(doc); } } } } for (const docs of docsByMirrorHash.values()) { assertInferenceActive(); docs.sort((left, right) => { if (left.uri < right.uri) return -1; if (left.uri > right.uri) return 1; return left.docid < right.docid ? -1 : left.docid > right.docid ? 1 : 0; }); } const collectionPaths = new Map(); if (collectionsResult.ok) { for (const c of collectionsResult.value) { assertInferenceActive(); collectionPaths.set(c.name, c.path); } } // Pre-fetch all chunks in one batch query (eliminates N+1) const chunksMapResult = await hydration.getChunksBatch([...neededHashes]); if (!chunksMapResult.ok) { return err("QUERY_FAILED", chunksMapResult.error.message); } const chunksMap = chunksMapResult.value; const getChunk = createChunkLookup(chunksMap); // Cache full content by mirrorHash for --full mode const contentCache = new Map< string, Awaited> >(); const results: SearchResult[] = []; const assemblyLimit = recencySort ? limit * 3 : limit; const docidMap = new Map(); // Track seen docids for --full de-duplication const seenDocids = new Set(); // Iterate until we have enough results (don't slice early - deduping may skip candidates) for (const [candidateIndex, candidate] of filteredCandidates.entries()) { assertInferenceActive(); // Stop when we have enough results if (!auxiliaryRankingActive && results.length >= assemblyLimit) { break; } // Find document from pre-fetched map const candidateDocs = ( docsByMirrorHash.get(candidate.mirrorHash) ?? [] ).filter( (doc) => candidate.documentId === undefined || candidate.documentId === doc.id ); if (candidateDocs.length === 0) { continue; } const docChunks = chunksMap.get(candidate.mirrorHash) ?? []; // Get chunk via O(1) lookup // For doc-level FTS (seq=0), fall back to first available chunk if exact lookup fails let chunk = getChunk(candidate.mirrorHash, candidate.seq); if (!chunk && candidate.seq === 0) { // Doc-level FTS uses seq=0 as placeholder - try first chunk const docChunks = chunksMap.get(candidate.mirrorHash); chunk = docChunks?.[0]; } if (!chunk) { continue; } // STRICT --lang filter: require exact match (excludes null/undefined) if (options.lang && chunk.language !== options.lang) { continue; } // For --full mode, fetch full mirror content const snippetChunk = options.full || !options.intent?.trim() ? chunk : (selectBestChunkForSteering( (chunksMap.get(candidate.mirrorHash) ?? []).filter( (chunk) => !options.lang || chunk.language === options.lang ), query, options.intent, { preferredSeq: chunk.seq, intentWeight: 0.3, } ) ?? chunk); let snippet = snippetChunk.text; let snippetStartLine = snippetChunk.startLine; let snippetRange: { startLine: number; endLine: number } | undefined = { startLine: snippetChunk.startLine, endLine: snippetChunk.endLine, }; if (options.full) { // Get or fetch full content for this mirrorHash let contentResult = contentCache.get(candidate.mirrorHash); if (!contentResult) { contentResult = await hydration.getContent(candidate.mirrorHash); contentCache.set(candidate.mirrorHash, contentResult); } if (contentResult.ok && contentResult.value) { snippet = contentResult.value; snippetRange = undefined; // Full content has no range } // Fallback to chunk text if content unavailable } else { const cleanedSnippet = cleanDisplaySnippet( snippetChunk.text, snippetChunk.text ); snippet = cleanedSnippet.text; snippetStartLine = snippetChunk.startLine + cleanedSnippet.startLineOffset; snippetRange = { startLine: snippetStartLine, endLine: snippetChunk.endLine, }; } for (const doc of candidateDocs) { assertInferenceActive(); if (!auxiliaryRankingActive && results.length >= assemblyLimit) break; const filterEval = evaluateDocumentChunkFilters( query, doc, docChunks, options ); if ( !filterEval.matches || (options.full && !auxiliaryRankingActive && seenDocids.has(doc.docid)) ) { continue; } const docidKey = `${candidate.mirrorHash}:${candidate.seq}${candidate.documentId === undefined ? "" : `:${candidate.documentId}`}`; if (!docidMap.has(docidKey)) docidMap.set(docidKey, doc.docid); const collectionPath = collectionPaths.get(doc.collection); if (options.full && !auxiliaryRankingActive) { seenDocids.add(doc.docid); } const baseResult: SearchResult = { docid: doc.docid, score: candidate.blendedScore, uri: doc.uri, title: doc.title ?? undefined, contentType: doc.contentType ?? undefined, categories: doc.categories ?? undefined, line: snippetStartLine, snippet, snippetLanguage: chunk.language ?? undefined, snippetRange, source: { relPath: doc.recordSourcePath ?? doc.relPath, absPath: collectionPath ? `${collectionPath}/${doc.recordSourcePath ?? doc.relPath}` : undefined, mime: doc.sourceMime, ext: doc.sourceExt, modifiedAt: doc.sourceMtime, documentDate: doc.frontmatterDate ?? undefined, sizeBytes: doc.sourceSize, sourceHash: doc.sourceHash, }, conversion: { mirrorHash: candidate.mirrorHash, converterId: doc.converterId ?? undefined, converterVersion: doc.converterVersion ?? undefined, }, record: projectRecordEvidenceMetadata(doc), }; const auxiliaryBaseScore = auxiliaryBaseScores.get( `${candidate.mirrorHash}:${candidate.seq}:${candidate.documentId ?? ""}` ) ?? candidate.blendedScore; const candidateKey = `${candidate.mirrorHash}:${candidate.seq}:${candidate.documentId ?? ""}`; const composedBeforeRerank = preRerankAdjustedCandidates.has(candidateKey); const scoringBaseScore = rerankResult.reranked && !composedBeforeRerank ? candidate.blendedScore : auxiliaryBaseScore; const scored = scoreContentTypeBoost( scoringBaseScore, doc.contentType ?? undefined, doc.contentTypeSource, doc.relPath, doc.collection, contentTypeRules, options.projectAffinity, { kind: "hybrid_blended", score: scoringBaseScore } ); const scoredResult = attachAuxiliaryScoreMetadata( baseResult, scored, rerankResult.reranked && composedBeforeRerank ? candidate.blendedScore : scored.projectAffinity.finalScore, hasProjectAffinity(options.projectAffinity) ); if (scoredResult.score < minScore) continue; results.push( attachSearchResultPlannerMetadata(scoredResult, { retrievalRank: candidateIndex + 1, mirrorHash: candidate.mirrorHash, seq: snippetChunk.seq, ...(auxiliaryRankingActive && snippetChunk.seq !== candidate.seq ? { retrievalSeq: candidate.seq } : {}), sources: [...candidate.sources].sort(), graphExpanded: candidate.sources.includes("graph"), startLine: snippetChunk.startLine, endLine: snippetChunk.endLine, passageHash: new Bun.CryptoHasher("sha256") .update(snippetChunk.text) .digest("hex"), }) ); } } timings.assemblyMs = performance.now() - assemblyStartedAt; timings.totalMs = performance.now() - runStartedAt; explainLines.push(explainTimings(timings)); explainLines.push(explainCounters(counters)); // ───────────────────────────────────────────────────────────────────────── // 6. Build explain data (if requested) // ───────────────────────────────────────────────────────────────────────── // ───────────────────────────────────────────────────────────────────────── // 7. Return results // ───────────────────────────────────────────────────────────────────────── const dedupedResults = options.full && auxiliaryRankingActive ? dedupeFullResultsByDocid(results) : results; if (recencySort) { dedupedResults.sort((a, b) => { const aTs = resolveRecencyTimestamp( a.source.documentDate, a.source.modifiedAt ); const bTs = resolveRecencyTimestamp( b.source.documentDate, b.source.modifiedAt ); if (aTs !== bTs) { return bTs - aTs; } return b.score - a.score; }); } else if (auxiliaryRankingActive && !rerankResult.reranked) { sortByFinalScoreStable(dedupedResults); } const finalResults = dedupedResults.slice(0, limit); for (const [index, result] of finalResults.entries()) { assertInferenceActive(); const metadata = result[SEARCH_RESULT_PLANNER_METADATA]; if (metadata) metadata.retrievalRank = index + 1; } const explainData = options.explain ? { lines: explainLines, results: auxiliaryRankingActive ? buildExplainResults(filteredCandidates, docidMap, finalResults) : buildExplainResults(filteredCandidates.slice(0, limit), docidMap), } : undefined; await attachSearchResultContexts(store, finalResults); const lineageResult = await attachSearchResultEgressLineage( store, finalResults, { ownershipHashes: [...neededHashes], ownershipDocuments: options.collection === undefined ? documents : undefined, collections: collectionsResult.ok ? collectionsResult.value : undefined, } ); if (!lineageResult.ok) { return err("QUERY_FAILED", lineageResult.error.message); } const output: SearchResults = { results: finalResults, meta: { query, mode: vectorsUsed ? "hybrid" : "bm25_only", expanded: expansion !== null, reranked: rerankResult.reranked, vectorsUsed, totalResults: finalResults.length, intent: options.intent, exclude: options.exclude, collection: options.collection, lang: options.lang, since: temporalRange.since, until: temporalRange.until, categories: options.categories, author: options.author, ...(options.filter ? { warnings: await typedMetadataWarnings(store, query, options) } : {}), candidateLimit, graphExpansion: { enabled: graphExpansion.meta.enabled, seedCount: graphExpansion.meta.seedCount, candidateCount: graphExpansion.meta.candidateCount, maxCandidates: graphExpansion.meta.maxCandidates, edgeConfidence: graphExpansion.meta.edgeConfidence, fallbackReasons: graphExpansion.meta.fallbackReasons, }, queryLanguage, queryModes: queryModeSummary, explain: explainData, trace: diagnoseTrace, }, }; const fallbackCodes = [...new Set(counters.fallbackEvents)].sort(); const capabilityOutcomes = [ { capability: "lexical_search", status: "used" as const }, vectorAvailable ? !vectorsUsed ? { capability: "semantic_search", status: "failed" as const, reasonCode: fallbackCodes.includes("vector_embed_error") ? "vector_embed_error" : "vector_search_error", } : { capability: "semantic_search", status: "used" as const } : { capability: "semantic_search", status: "unavailable" as const, reasonCode: "vector_unavailable", }, expansion !== null ? { capability: "query_expansion", status: "used" as const } : { capability: "query_expansion", status: expansionStatus === "attempted" ? ("failed" as const) : ("unavailable" as const), reasonCode: expansionStatus === "attempted" ? "expansion_error" : expansionStatus === "skipped_strong" ? "expansion_skipped_strong" : "expansion_disabled", }, rerankResult.reranked ? { capability: "reranking", status: "used" as const } : { capability: "reranking", status: rerankResult.fallbackReason === "error" ? ("failed" as const) : ("unavailable" as const), reasonCode: rerankResult.fallbackReason === "error" ? "rerank_error" : "rerank_disabled", }, graphExpansion.meta.enabled ? { capability: "graph_expansion", status: "used" as const } : { capability: "graph_expansion", status: "unavailable" as const, reasonCode: graphExpansion.meta.fallbackReasons[0] ?? "graph_disabled", }, ]; attachSearchResultsTraceMetadata(output, { capabilityOutcomes, fallbackCodes, }); const traceResult = await options.traceSession?.recordRetrieval( output, timings.totalMs ); if (traceResult && !traceResult.ok) { return err( "QUERY_FAILED", `Trace recording failed: ${traceResult.error.message}`, traceResult.error.cause ); } return ok(output); } function dedupeFullResultsByDocid(results: SearchResult[]): SearchResult[] { const bestByDocid = new Map(); for (const result of results) { assertInferenceActive(); const existing = bestByDocid.get(result.docid); if (!existing || result.score > existing.score) { bestByDocid.set(result.docid, result); } } return [...bestByDocid.values()]; }