{"version":3,"file":"rerank.d.ts","sourceRoot":"","sources":["../../../src/core/search/rerank.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAKH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AA6GjD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAQnD;AAiCD,MAAM,WAAW,YAAY;IAC5B,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAiBlH;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,cAAc,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,YAAY,CAqFhH","sourcesContent":["/**\n * Deterministic reranker over the fused top-50\n * (docs/hybrid-retrieval-design.md, step 7 of the shipping order).\n *\n * The eval gate showed fused Recall@50 well above Recall@5/10 — the right\n * candidates survive fusion but sit too deep. This reranker re-orders them\n * using evidence that is only cheap to compute *after* fusion, when there\n * are ≤50 candidates instead of thousands of lines:\n *\n *   - term coverage: how many distinct query terms appear in the candidate's\n *     actual expanded window (read from disk);\n *   - path affinity: query terms appearing in the candidate's file path —\n *     this is what lets a query like `core/search/hybrid-search.ts` rank the\n *     file itself first, which content grep alone cannot do;\n *   - fused prior: the RRF ordering, so retriever consensus still counts.\n *\n * Not every signal suits every query. A query that names something and a query\n * that describes behaviour want different evidence, so the name-matching\n * signal is gated on {@link queryIsProse} — see its comment for the numbers.\n *\n * Purely lexical-statistical and deterministic — no model, no I/O beyond\n * reading candidate windows. A cross-encoder can later replace the scoring\n * function behind the same signature; that model work belongs to\n * `kolisachint/embeddingsearchtools`, not here.\n */\n\nimport { readFileSync } from \"fs\";\nimport path from \"path\";\nimport { buildLexicalQueryPlan } from \"./lexical-retriever.js\";\nimport type { FusedCandidate } from \"./types.js\";\n\n/** Weights of the scoring blend. The eval harness (`bun run search-eval`) is\n *  the instrument for changing them — don't tune blind. */\nconst WEIGHT_FUSED_PRIOR = 0.4;\nconst WEIGHT_TERM_COVERAGE = 0.35;\nconst WEIGHT_PATH_AFFINITY = 0.25;\n/** Additive bonus when the query *is* the candidate's path (or its suffix):\n *  the caller named the file, so no amount of content evidence elsewhere\n *  should outrank it. */\nconst EXACT_PATH_BONUS = 0.5;\n/**\n * Additive bonus when the window *declares* a query term rather than merely\n * mentioning it.\n *\n * This targets the largest measured gap in the eval: on the 22 exact-symbol\n * queries the definition is in the top 10 about 85% of the time but ranked\n * first only about 20% of the time. Call sites outnumber definitions and\n * contain the identical identifier, so term coverage — which saturates at 1.0\n * for both — cannot separate them. Structure can.\n */\nconst DECLARATION_BONUS = 0.3;\n\n/**\n * Sentence glue. Identifiers and paths never contain these words, so two or\n * more of them means the query is a sentence rather than a name.\n *\n * This gates {@link DECLARATION_BONUS} because that bonus is a *name-matching*\n * signal and a prose query has no name to match. Its plan terms are ordinary\n * words — \"results\", \"search\", \"index\" — so it fires on whichever candidate\n * happens to declare a variable by one of them, at a weight larger than the\n * entire path-affinity term, and buries the fused ordering that did know the\n * answer. On the conceptual class reranking was scoring *below not reranking\n * at all* (MRR 0.246 un-reranked vs 0.124 reranked on `semantic`).\n *\n * Measured on the 62-query gold set, gating it moved conceptual MRR +0.046\n * (`semantic +rr`), +0.040 (`auto +rr`), +0.056 (`bm25+dense +rr`) and left\n * exact-symbol, error-fragment and path bit-identical — the gate never fires\n * on those, by construction. Overall MRR +0.007 to +0.013, which the paired\n * sign test does not call significant (p = 0.11 to 0.29); the classes it\n * protects are what justify it, not the aggregate.\n *\n * Path affinity is deliberately *not* gated. It is a topic signal, not a name\n * signal: \"how does a grep line number become an embedding chunk id\" wants\n * files with `grep` and `chunk` in the path. Gating it too was measured and\n * was strictly worse — same conceptual gain, roughly double the cross-file\n * loss (−0.058 vs −0.032 on `semantic +rr`).\n *\n * Deliberately conservative: two hits, not one, so a terse query like\n * `hybrid search fusion` keeps today's scoring untouched.\n */\nconst PROSE_FUNCTION_WORDS = new Set([\n\t\"a\",\n\t\"after\",\n\t\"all\",\n\t\"an\",\n\t\"and\",\n\t\"any\",\n\t\"are\",\n\t\"as\",\n\t\"at\",\n\t\"be\",\n\t\"been\",\n\t\"before\",\n\t\"between\",\n\t\"but\",\n\t\"by\",\n\t\"can\",\n\t\"does\",\n\t\"do\",\n\t\"each\",\n\t\"for\",\n\t\"from\",\n\t\"had\",\n\t\"has\",\n\t\"have\",\n\t\"how\",\n\t\"if\",\n\t\"in\",\n\t\"into\",\n\t\"is\",\n\t\"it\",\n\t\"its\",\n\t\"of\",\n\t\"on\",\n\t\"one\",\n\t\"or\",\n\t\"should\",\n\t\"so\",\n\t\"than\",\n\t\"that\",\n\t\"the\",\n\t\"then\",\n\t\"this\",\n\t\"to\",\n\t\"under\",\n\t\"was\",\n\t\"were\",\n\t\"what\",\n\t\"when\",\n\t\"where\",\n\t\"which\",\n\t\"why\",\n\t\"with\",\n\t\"would\",\n]);\n/** Function words needed before a query counts as prose. */\nconst PROSE_WORD_THRESHOLD = 2;\n\n/**\n * Is `query` a sentence rather than a name?\n *\n * A quoted segment is never prose regardless of its words: the plan collapses\n * it to one literal term, so the declaration bonus cannot fire on it anyway,\n * and its path-token split is what lets `\"Theme not initialized…\"` find\n * `core/theme.ts`.\n */\nexport function queryIsProse(query: string): boolean {\n\tif (/[\"'`][^\"'`]+[\"'`]/.test(query)) return false;\n\tconst words = query.toLowerCase().match(/[a-z]+/g) ?? [];\n\tlet hits = 0;\n\tfor (const word of words) {\n\t\tif (PROSE_FUNCTION_WORDS.has(word) && ++hits >= PROSE_WORD_THRESHOLD) return true;\n\t}\n\treturn false;\n}\n\n/** Keywords that introduce a definition across the languages this indexes.\n *  Matched against lowercased text, so the term is lowercased too. */\nconst DECLARATION_KEYWORDS =\n\t\"function|class|interface|type|enum|struct|impl|trait|fn|def|const|let|var|namespace|module\";\n\n/** Does `window` declare `term`, as opposed to referencing it? */\nfunction declaresTerm(window: string, term: string): boolean {\n\tconst escaped = term.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n\t// `function foo(`, `class Foo {`, `const foo =` ...\n\tif (new RegExp(`\\\\b(?:${DECLARATION_KEYWORDS})\\\\s+${escaped}\\\\b`).test(window)) return true;\n\t// `foo(...) {` at the start of a line — methods, Go/Rust receivers, Python defs\n\t// already covered above, but this catches object-literal and class members.\n\tif (\n\t\tnew RegExp(`^\\\\s*(?:(?:async|public|private|protected|static|export)\\\\s+)*${escaped}\\\\s*[(<]`, \"m\").test(window)\n\t) {\n\t\treturn true;\n\t}\n\treturn false;\n}\n\n/** Inverse document frequency over the candidate pool.\n *\n * True corpus IDF lives in the BM25 index and is not exposed over the daemon\n * protocol, so this approximates it with the candidate set: a term present in\n * every candidate discriminates nothing, one present in three carries the\n * signal. That is the comparison the reranker actually needs to make, since it\n * only ever orders candidates against each other. */\nfunction inverseDocumentFrequency(documentFrequency: number, total: number): number {\n\treturn Math.log(1 + total / Math.max(1, documentFrequency));\n}\n\nexport interface RerankResult {\n\tcandidates: FusedCandidate[];\n\tlatencyMs: number;\n}\n\n/**\n * The exact source text each candidate stands for, as the model should see it.\n *\n * Shared with the cross-encoder path so both rerankers score identical text —\n * otherwise a comparison between them would partly measure which one got a\n * better view of the candidate.\n */\nexport function readCandidateWindows(candidates: readonly FusedCandidate[], cwd: string): Array<string | undefined> {\n\tconst fileCache = new Map<string, string[] | undefined>();\n\tconst read = (rel: string): string[] | undefined => {\n\t\tif (!fileCache.has(rel)) {\n\t\t\ttry {\n\t\t\t\tfileCache.set(rel, readFileSync(path.resolve(cwd, rel), \"utf-8\").split(\"\\n\"));\n\t\t\t} catch {\n\t\t\t\tfileCache.set(rel, undefined);\n\t\t\t}\n\t\t}\n\t\treturn fileCache.get(rel);\n\t};\n\treturn candidates.map((candidate) => {\n\t\tconst lines = read(candidate.path);\n\t\tif (!lines) return undefined;\n\t\treturn lines.slice(Math.max(0, candidate.startLine - 1), Math.min(lines.length, candidate.endLine)).join(\"\\n\");\n\t});\n}\n\nexport function rerankCandidates(query: string, candidates: readonly FusedCandidate[], cwd: string): RerankResult {\n\tconst startedMs = Date.now();\n\tconst plan = buildLexicalQueryPlan(query);\n\tif (!plan || candidates.length < 2) {\n\t\treturn { candidates: [...candidates], latencyMs: Date.now() - startedMs };\n\t}\n\tconst terms = plan.terms;\n\tconst queryPath = query.trim().toLowerCase();\n\t// Prose asks about behaviour, not about a name, so the one signal that reads\n\t// candidates *as names* is switched off.\n\tconst prose = queryIsProse(query);\n\n\tconst fileCache = new Map<string, string[] | undefined>();\n\tconst readLines = (rel: string): string[] | undefined => {\n\t\tif (!fileCache.has(rel)) {\n\t\t\ttry {\n\t\t\t\tconst content = readFileSync(path.resolve(cwd, rel), \"utf-8\");\n\t\t\t\tfileCache.set(rel, content.toLowerCase().split(\"\\n\"));\n\t\t\t} catch {\n\t\t\t\tfileCache.set(rel, undefined);\n\t\t\t}\n\t\t}\n\t\treturn fileCache.get(rel);\n\t};\n\n\t// Read every candidate window once: the term/declaration signals and the\n\t// candidate-pool IDF all need them, and files repeat across candidates.\n\tconst windows = candidates.map((candidate) => {\n\t\tconst lines = readLines(candidate.path);\n\t\tif (!lines) return undefined;\n\t\treturn lines.slice(Math.max(0, candidate.startLine - 1), Math.min(lines.length, candidate.endLine)).join(\"\\n\");\n\t});\n\n\t// Candidate-pool document frequency per term, for the IDF weighting below.\n\tconst documentFrequency = new Map<string, number>();\n\tfor (const term of terms) {\n\t\tdocumentFrequency.set(term, windows.filter((w) => w?.includes(term)).length);\n\t}\n\tconst termWeight = new Map(\n\t\tterms.map((t) => [t, inverseDocumentFrequency(documentFrequency.get(t) ?? 0, candidates.length)]),\n\t);\n\tconst totalTermWeight = terms.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0);\n\n\t// Fused prior normalized by score, not by position: a candidate both\n\t// retrievers agreed on should outrank one that squeaked in, and a uniform\n\t// 1 - index/length ramp throws that magnitude away.\n\tconst maxRrfScore = Math.max(...candidates.map((c) => c.rrfScore), Number.MIN_VALUE);\n\n\tconst scored = candidates.map((candidate, index) => {\n\t\tconst fusedPrior = candidate.rrfScore / maxRrfScore;\n\n\t\tconst window = windows[index];\n\t\tlet termCoverage = 0;\n\t\tlet declaresAnyTerm = false;\n\t\tif (window && terms.length > 0) {\n\t\t\tconst present = terms.filter((t) => window.includes(t));\n\t\t\ttermCoverage =\n\t\t\t\ttotalTermWeight > 0\n\t\t\t\t\t? present.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0) / totalTermWeight\n\t\t\t\t\t: present.length / terms.length;\n\t\t\tdeclaresAnyTerm = !prose && present.some((t) => declaresTerm(window, t));\n\t\t}\n\n\t\tconst lowerPath = candidate.path.toLowerCase();\n\t\t// A quoted phrase rarely names a file; split it into path-ish tokens so\n\t\t// `\"token budget exceeded\"` still gets partial path credit.\n\t\tconst pathTerms = terms.length === 1 ? terms[0].split(/[^a-z0-9_$]+/).filter((t) => t.length >= 3) : terms;\n\t\tconst pathAffinity =\n\t\t\tpathTerms.length > 0 ? pathTerms.filter((t) => lowerPath.includes(t)).length / pathTerms.length : 0;\n\n\t\tconst exactPath =\n\t\t\tqueryPath.length >= 3 && (lowerPath === queryPath || lowerPath.endsWith(`/${queryPath}`)) ? 1 : 0;\n\n\t\tconst score =\n\t\t\tWEIGHT_FUSED_PRIOR * fusedPrior +\n\t\t\tWEIGHT_TERM_COVERAGE * termCoverage +\n\t\t\tWEIGHT_PATH_AFFINITY * pathAffinity +\n\t\t\tEXACT_PATH_BONUS * exactPath +\n\t\t\t(declaresAnyTerm ? DECLARATION_BONUS : 0);\n\t\treturn { candidate, index, score };\n\t});\n\n\t// Stable, deterministic: score desc, fused order as tie-break.\n\tscored.sort((a, b) => b.score - a.score || a.index - b.index);\n\treturn { candidates: scored.map((s) => s.candidate), latencyMs: Date.now() - startedMs };\n}\n"]}