{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEhF,MAAM,WAAW,YAAY;IAC5B,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAC;IACX;6DACyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,YAAY,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;iEAC6D;IAC7D,YAAY,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,YAAY,EAAE,SAAS,UAAU,EAsC7C,CAAC;AAKF,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,GAAG,OAAO,CAKhF;AAED,gFAAgF;AAChF,wBAAgB,SAAS,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,YAAY,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAQhH;AAED;;;;;;;;GAQG;AACH,wBAAgB,GAAG,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,YAAY,EAAE,GAAG,MAAM,CAQ/F;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,kBAAkB,CAAC;IACjC,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,wBAAsB,aAAa,CAClC,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,EACpB,OAAO,GAAE,SAAS,UAAU,EAAiB,EAC7C,OAAO,CAAC,EAAE,gBAAgB,EAC1B,aAAa,CAAC,EAAE,gBAAgB,GAC9B,OAAO,CAAC,eAAe,EAAE,CAAC,CAqC5B","sourcesContent":["/**\n * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the\n * shipping order).\n *\n * Measures Recall@K and MRR for lexical, semantic, and hybrid retrieval across\n * an RRF `k` sweep, against a gold set keyed by **path + line range matched by\n * span overlap** — never by chunkId, which is only stable per index build.\n * Recall@50 doubles as the reranker gate: a gold span that never reaches the\n * fused top-50 cannot be rescued by any reranker.\n *\n * Rank-sensitive metrics are the point of Recall@1 and MRR: an agent reads the\n * top result or two, so \"the gold span is somewhere in the top 5\" understates\n * how much ordering matters. Recall@5/10/50 are kept for continuity with the\n * numbers already published in the design note.\n *\n * Driven by `scripts/search-eval.ts` via `eval-harness.ts`; the scoring logic\n * lives here so it is unit-testable without an embedding index.\n */\n\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { retrieveCandidates } from \"./hybrid-search.js\";\nimport type { CandidateSpan, ResolvedSearchMode, SearchMode } from \"./types.js\";\n\nexport interface EvalGoldSpan {\n\t/** Repo-relative POSIX path. */\n\tpath: string;\n\t/** 1-based inclusive; omit both to accept any span in the file. */\n\tstartLine?: number;\n\tendLine?: number;\n\t/**\n\t * Literal source text that must occur inside `[startLine, endLine]`. Not\n\t * used for scoring — it is how the gold set survives the corpus moving\n\t * underneath it: `scripts/search-eval-gold.ts` re-resolves the line range\n\t * from this anchor, and the fixture test fails when an anchor no longer\n\t * sits inside its recorded range. Omitted for `scope: \"file\"` entries.\n\t */\n\tanchor?: string;\n\t/**\n\t * `\"span\"` (default) scores by overlap with the recorded line range;\n\t * `\"file\"` accepts any span in the file. File scope is deliberate for\n\t * path-class queries, where the whole file *is* the answer — it is\n\t * recorded explicitly so a file-level score is never mistaken for a\n\t * span-level one.\n\t */\n\tscope?: \"span\" | \"file\";\n}\n\nexport interface EvalQuery {\n\tid: string;\n\t/** Query class from the design doc (exact-symbol, path, error-fragment,\n\t *  conceptual, cross-file, boundary). Reporting only. */\n\tclass: string;\n\tquery: string;\n\tgold: EvalGoldSpan[];\n}\n\nexport interface EvalConfig {\n\tlabel: string;\n\tmode: SearchMode;\n\trrfK?: number;\n\trerank?: boolean;\n\t/**\n\t * Score this config against a daemon-side hybrid store (BM25 fused with\n\t * vectors inside the Rust daemon) instead of the dense-only index. Skipped\n\t * when the harness has no hybrid service, so records without one simply\n\t * omit the row rather than silently scoring it as plain semantic.\n\t */\n\tdaemonHybrid?: boolean;\n\t/**\n\t * Fetch the daemon's BM25 index as a separate leg and fuse it here with\n\t * dense and grep. Like `daemonHybrid` it needs a hybrid store, so it is\n\t * skipped when the harness has no hybrid service.\n\t */\n\tbm25Leg?: boolean;\n\t/** Rerank with the daemon's cross-encoder rather than the deterministic\n\t *  scorer. Needs a service, so it is skipped without one. */\n\tcrossEncoder?: boolean;\n}\n\n/**\n * The sweep from the design doc — single retrievers, hybrid across k, the\n * routed auto mode — plus reranked (`+rr`) variants for the step 7 gate.\n *\n * Exported because `scripts/search-eval.ts` and `test/search-eval.test.ts`\n * both consume it. It previously lost its `export` to a knip-driven\n * dead-export sweep (02efaab): the only importer was a `.mjs` script reading\n * from `dist/`, which static analysis cannot see, and the eval gate silently\n * stopped running. The TypeScript importers are the fix — do not \"clean up\"\n * this export without checking them.\n */\nexport const EVAL_CONFIGS: readonly EvalConfig[] = [\n\t{ label: \"lexical\", mode: \"lexical\" },\n\t{ label: \"semantic\", mode: \"semantic\" },\n\t{ label: \"hybrid k=0\", mode: \"hybrid\", rrfK: 0 },\n\t{ label: \"hybrid k=2\", mode: \"hybrid\", rrfK: 2 },\n\t{ label: \"hybrid k=10\", mode: \"hybrid\", rrfK: 10 },\n\t{ label: \"hybrid k=60\", mode: \"hybrid\", rrfK: 60 },\n\t{ label: \"auto\", mode: \"auto\" },\n\t{ label: \"lexical +rr\", mode: \"lexical\", rerank: true },\n\t{ label: \"semantic +rr\", mode: \"semantic\", rerank: true },\n\t{ label: \"hybrid k=2 +rr\", mode: \"hybrid\", rrfK: 2, rerank: true },\n\t{ label: \"hybrid k=60 +rr\", mode: \"hybrid\", rrfK: 60, rerank: true },\n\t{ label: \"auto +rr\", mode: \"auto\", rerank: true },\n\t// Is BM25 a better lexical leg than ripgrep? These two run the daemon's own\n\t// vector+BM25 fusion and no ripgrep at all, so comparing them against\n\t// \"semantic\" isolates what BM25 adds, and against \"hybrid k=2\" compares the\n\t// two lexical legs at the system level.\n\t{ label: \"daemon-hybrid\", mode: \"semantic\", daemonHybrid: true },\n\t{ label: \"daemon-hybrid +rr\", mode: \"semantic\", rerank: true, daemonHybrid: true },\n\t// Three-way fusion: dense + BM25 + grep, all fused here so `k` is ours and\n\t// the per-leg ranks survive into the trace. `bm25+dense` isolates what the\n\t// grep leg still contributes once BM25 is present.\n\t//\n\t// `3-way` used to lose badly to `bm25+dense` (MRR 0.608 -> 0.552, 18 of 62\n\t// queries worse) because two lexical views of the same corpus dilute each\n\t// other in fusion. Its grep leg is now scoped to files the index has not\n\t// read, so the two rows are identical on this corpus by construction —\n\t// the difference only appears under `--live-edits`, where `3-way` scores\n\t// 1.000 and `bm25+dense` scores 0.000.\n\t{ label: \"bm25+dense +rr\", mode: \"semantic\", rerank: true, bm25Leg: true },\n\t// The shipped default: BM25 over the indexed corpus, grep narrowed to what\n\t// the index has not read, dense alongside both.\n\t{ label: \"3-way +rr\", mode: \"hybrid\", rerank: true, bm25Leg: true },\n\t// Cross-encoder reranking, against the same retrieval each deterministic\n\t// `+rr` row uses — so the delta is the reranker and nothing else.\n\t{ label: \"semantic +ce\", mode: \"semantic\", crossEncoder: true },\n\t{ label: \"auto +ce\", mode: \"auto\", crossEncoder: true },\n\t{ label: \"bm25+dense +ce\", mode: \"semantic\", bm25Leg: true, crossEncoder: true },\n];\n\n/** Candidates fetched per eval query — deep enough for the reranker gate. */\nconst EVAL_FETCH_LIMIT = 50;\n\nexport function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean {\n\tif (span.path !== gold.path) return false;\n\tif (gold.scope === \"file\") return true;\n\tif (gold.startLine === undefined || gold.endLine === undefined) return true;\n\treturn span.startLine <= gold.endLine && span.endLine >= gold.startLine;\n}\n\n/** Fraction of gold spans matched by at least one of the top-`k` candidates. */\nexport function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number {\n\tif (gold.length === 0) return 0;\n\tconst top = candidates.slice(0, k);\n\tlet matched = 0;\n\tfor (const g of gold) {\n\t\tif (top.some((c) => spanMatchesGold(c, g))) matched++;\n\t}\n\treturn matched / gold.length;\n}\n\n/**\n * Mean reciprocal rank, averaged over gold spans: for each gold span, `1/rank`\n * of the first candidate matching it (0 when it never appears).\n *\n * Averaging per gold span rather than taking the single first hit keeps\n * multi-span queries (the cross-file class) honest — finding one of two\n * required sites should not score like finding both. For single-span queries\n * this reduces to textbook MRR.\n */\nexport function mrr(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[]): number {\n\tif (gold.length === 0) return 0;\n\tlet total = 0;\n\tfor (const g of gold) {\n\t\tconst index = candidates.findIndex((c) => spanMatchesGold(c, g));\n\t\tif (index >= 0) total += 1 / (index + 1);\n\t}\n\treturn total / gold.length;\n}\n\nexport interface EvalQueryResult {\n\tlabel: string;\n\tresolvedMode: ResolvedSearchMode;\n\tdegraded: boolean;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n}\n\nexport async function evaluateQuery(\n\tcwd: string,\n\tevalQuery: EvalQuery,\n\tconfigs: readonly EvalConfig[] = EVAL_CONFIGS,\n\tservice?: EmbsearchService,\n\thybridService?: EmbsearchService,\n): Promise<EvalQueryResult[]> {\n\tconst results: EvalQueryResult[] = [];\n\tfor (const config of configs) {\n\t\t// A daemon-hybrid config against the plain store would error in the\n\t\t// daemon (`query_hybrid requires a hybrid store`); omit the row instead.\n\t\tif ((config.daemonHybrid || config.bm25Leg) && !hybridService) continue;\n\t\t// Cross-encoder rows need a live daemon; omit rather than silently\n\t\t// falling back to the deterministic reranker under a \"+ce\" label.\n\t\tif (config.crossEncoder && !(config.bm25Leg ? hybridService : service)?.supportsCrossEncoder()) continue;\n\t\tconst retrieved = await retrieveCandidates({\n\t\t\tcwd,\n\t\t\tquery: evalQuery.query,\n\t\t\tmode: config.mode,\n\t\t\trrfK: config.rrfK,\n\t\t\trerank: config.rerank ?? false,\n\t\t\tlimit: EVAL_FETCH_LIMIT,\n\t\t\tservice: config.daemonHybrid || config.bm25Leg ? hybridService : service,\n\t\t\tdaemonHybrid: config.daemonHybrid,\n\t\t\t// Explicit, never inherited: `bm25Leg` now defaults to on in production,\n\t\t\t// and letting that leak in would silently give every historical row a\n\t\t\t// BM25 leg it never had — the numbers would stop meaning what their\n\t\t\t// labels say.\n\t\t\tbm25Leg: config.bm25Leg ?? false,\n\t\t\tcrossEncoder: config.crossEncoder,\n\t\t});\n\t\tresults.push({\n\t\t\tlabel: config.label,\n\t\t\tresolvedMode: retrieved.resolvedMode,\n\t\t\tdegraded: retrieved.degradedReason !== undefined,\n\t\t\trecallAt1: recallAtK(retrieved.candidates, evalQuery.gold, 1),\n\t\t\trecallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),\n\t\t\trecallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),\n\t\t\trecallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),\n\t\t\tmrr: mrr(retrieved.candidates, evalQuery.gold),\n\t\t});\n\t}\n\treturn results;\n}\n"]}