{"version":3,"file":"lexical.d.ts","sourceRoot":"","sources":["../../../src/core/capabilities/lexical.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA4BnD;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAmB/C;AAsED,MAAM,WAAW,UAAU;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACd;AAQD;;;;GAIG;AACH,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;IACrD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,YAAY,IAAI,EAAE,SAAS,aAAa,EAAE,EAWzC;IAED,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,qFAAqF;IACrF,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,SAAK,GAAG,UAAU,EAAE,CA+B1C;CACD","sourcesContent":["/**\n * In-memory BM25 over the capability index — the leg that always works.\n *\n * Deliberately not the repo's lexical retriever: that one shells out to ripgrep\n * over files on disk, and capabilities are a few hundred short strings held in\n * memory. It is also not optional. The dense leg needs a binary that may not be\n * installed and a store that may still be building, so if retrieval depended on\n * it, \"find me a tool that sends email\" would work on some machines and not\n * others. This leg makes the floor deterministic and dependency-free; dense is\n * strictly additive on top.\n *\n * The tokenizer does the load-bearing work here. Tool names are the query terms\n * that matter most and they arrive as `mcp_github_create_pull_request` or\n * `createPullRequest`, so a naive whitespace split would make the single most\n * common query shape — a name the model half-remembers — the one thing BM25\n * cannot match.\n */\n\nimport type { CapabilityDoc } from \"./registry.js\";\n\n/** Standard Okapi BM25 parameters. Nothing here justifies tuning them. */\nconst K1 = 1.2;\nconst B = 0.75;\n\n/**\n * Fold a regular English plural to its singular, or return undefined.\n *\n * Deliberately crude: only the endings that are unambiguous without a\n * dictionary. Documentation headings are written in whichever number reads best\n * (\"Themes\", \"Custom Providers\") while questions are asked in the other (\"how\n * do I add a theme\"), and with no folding at all those two never meet — the\n * single most useful term in the query is the one term that cannot match.\n *\n * Irregulars are left alone. Getting \"indices\" wrong costs a missed hit;\n * inventing a stemmer that mangles \"status\" into \"statu\" would cost matches\n * that work today.\n */\nfunction singularize(token: string): string | undefined {\n\tif (token.length > 4 && token.endsWith(\"ies\")) return `${token.slice(0, -3)}y`;\n\tif (token.length > 4 && /(?:ss|sh|ch|x|z)es$/.test(token)) return token.slice(0, -2);\n\tif (token.length > 3 && token.endsWith(\"s\") && !token.endsWith(\"ss\") && !token.endsWith(\"us\")) {\n\t\treturn token.slice(0, -1);\n\t}\n\treturn undefined;\n}\n\n/**\n * Split identifiers the way a person reads them: `mcp_github_create_pr` and\n * `createPullRequest` both yield their parts, and the original token is kept so\n * an exact name still scores as an exact match.\n *\n * Singular forms are emitted *alongside* the originals rather than replacing\n * them, which is the same bargain the identifier splitting makes: an exact\n * token still matches exactly, and a near miss now matches too.\n */\nexport function tokenize(text: string): string[] {\n\tconst out: string[] = [];\n\tconst push = (token: string): void => {\n\t\tout.push(token);\n\t\tconst singular = singularize(token);\n\t\tif (singular && singular !== token) out.push(singular);\n\t};\n\tfor (const raw of text.toLowerCase().match(/[a-z0-9]+(?:[_-][a-z0-9]+)*/gi) ?? []) {\n\t\tconst token = raw.toLowerCase();\n\t\tpush(token);\n\t\t// Split on separators, then on camelCase boundaries in the source text.\n\t\tconst parts = token.split(/[_-]+/).filter(Boolean);\n\t\tif (parts.length > 1) for (const part of parts) push(part);\n\t}\n\tfor (const camel of text.match(/[a-z][a-z0-9]*|[A-Z][a-z0-9]*|[A-Z]+(?![a-z])/g) ?? []) {\n\t\tconst lower = camel.toLowerCase();\n\t\tif (lower.length > 1) push(lower);\n\t}\n\treturn out;\n}\n\n/**\n * Function words dropped from *queries* only.\n *\n * Questions arrive as \"how do I add a custom theme\", and in a corpus of a few\n * hundred short documents the filler carries real weight: a section whose prose\n * happens to say \"Add an AGENTS.md file ... to tell it how to work\" outscores\n * the section actually titled \"Creating a Custom Theme\", because it matched\n * three throwaway words to the target's one meaningful one.\n *\n * Query-side only, deliberately. Stripping these from documents too would\n * change every document length and every average, re-tuning a ranking that\n * works; dropping a term from the query just stops it contributing, which is\n * the whole intent.\n */\nconst QUERY_STOPWORDS = new Set([\n\t\"a\",\n\t\"an\",\n\t\"and\",\n\t\"are\",\n\t\"as\",\n\t\"at\",\n\t\"be\",\n\t\"by\",\n\t\"can\",\n\t\"do\",\n\t\"does\",\n\t\"for\",\n\t\"from\",\n\t\"get\",\n\t\"how\",\n\t\"i\",\n\t\"in\",\n\t\"is\",\n\t\"it\",\n\t\"its\",\n\t\"me\",\n\t\"my\",\n\t\"of\",\n\t\"on\",\n\t\"or\",\n\t\"so\",\n\t\"that\",\n\t\"the\",\n\t\"then\",\n\t\"there\",\n\t\"this\",\n\t\"to\",\n\t\"use\",\n\t\"using\",\n\t\"want\",\n\t\"was\",\n\t\"what\",\n\t\"when\",\n\t\"where\",\n\t\"which\",\n\t\"who\",\n\t\"why\",\n\t\"will\",\n\t\"with\",\n\t\"you\",\n\t\"your\",\n]);\n\n/** The text a document is matched on: name first, since that is what queries name. */\nfunction documentText(doc: CapabilityDoc): string {\n\treturn `${doc.name} ${doc.name} ${doc.source ?? \"\"} ${doc.description}`;\n}\n\nexport interface LexicalHit {\n\tid: string;\n\tscore: number;\n}\n\ninterface Posting {\n\tid: string;\n\tlength: number;\n\tcounts: Map<string, number>;\n}\n\n/**\n * A built BM25 index. Cheap enough to rebuild whenever the capability set\n * changes — a few hundred short documents — so there is no invalidation story\n * to get wrong.\n */\nexport class LexicalIndex {\n\tprivate readonly postings: Posting[] = [];\n\tprivate readonly docFreq = new Map<string, number>();\n\tprivate readonly avgLength: number;\n\n\tconstructor(docs: readonly CapabilityDoc[]) {\n\t\tlet total = 0;\n\t\tfor (const doc of docs) {\n\t\t\tconst tokens = tokenize(documentText(doc));\n\t\t\tconst counts = new Map<string, number>();\n\t\t\tfor (const t of tokens) counts.set(t, (counts.get(t) ?? 0) + 1);\n\t\t\tfor (const t of counts.keys()) this.docFreq.set(t, (this.docFreq.get(t) ?? 0) + 1);\n\t\t\tthis.postings.push({ id: doc.id, length: tokens.length, counts });\n\t\t\ttotal += tokens.length;\n\t\t}\n\t\tthis.avgLength = this.postings.length > 0 ? total / this.postings.length : 0;\n\t}\n\n\tget size(): number {\n\t\treturn this.postings.length;\n\t}\n\n\t/** Top `k` documents for `query`, best first. Documents scoring zero are omitted. */\n\tsearch(query: string, k = 10): LexicalHit[] {\n\t\tconst raw = tokenize(query);\n\t\t// Fall back to the unfiltered terms when a query is nothing but function\n\t\t// words, so \"what is it\" still searches rather than silently matching all.\n\t\tconst filtered = raw.filter((t) => !QUERY_STOPWORDS.has(t));\n\t\tconst terms = filtered.length > 0 ? filtered : raw;\n\t\tif (terms.length === 0 || this.postings.length === 0) return [];\n\t\tconst n = this.postings.length;\n\n\t\tconst hits: LexicalHit[] = [];\n\t\tfor (const posting of this.postings) {\n\t\t\tlet score = 0;\n\t\t\tfor (const term of new Set(terms)) {\n\t\t\t\tconst tf = posting.counts.get(term);\n\t\t\t\tif (!tf) continue;\n\t\t\t\tconst df = this.docFreq.get(term) ?? 0;\n\t\t\t\t// Okapi IDF, floored at zero: a term in every document carries no\n\t\t\t\t// signal, and the raw formula would make it actively negative.\n\t\t\t\tconst idf = Math.max(0, Math.log(1 + (n - df + 0.5) / (df + 0.5)));\n\t\t\t\tconst norm = tf * (K1 + 1);\n\t\t\t\tconst denom = tf + K1 * (1 - B + (B * posting.length) / (this.avgLength || 1));\n\t\t\t\tscore += idf * (norm / denom);\n\t\t\t}\n\t\t\tif (score > 0) hits.push({ id: posting.id, score });\n\t\t}\n\n\t\t// Ties break by id so the same query always returns the same order — a\n\t\t// retrieval tool that reshuffles equal-scoring results is a reproducibility\n\t\t// problem disguised as a ranking one.\n\t\thits.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));\n\t\treturn hits.slice(0, k);\n\t}\n}\n"]}