{"version":3,"file":"lexical-retriever.d.ts","sourceRoot":"","sources":["../../../src/core/search/lexical-retriever.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAWhD,MAAM,WAAW,gBAAgB;IAChC,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,KAAK,EAAE,MAAM,EAAE,CAAC;CAChB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAgBjF;AAED,0DAA0D;AAC1D,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAErE;AAUD;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1B;AAiBD,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAmH5F","sourcesContent":["/**\n * Internal lexical retriever for hybrid search.\n *\n * This is not the grep *tool* — it is the lexical recall backend: it turns a\n * natural query into a ripgrep pattern, streams matches, and returns bare\n * `rel:line` hits for the grep→chunk adapter. rg drives the fast path; the\n * pure-JS nativeGrep fallback keeps restricted environments working, same as\n * the grep tool.\n */\n\nimport { createInterface } from \"node:readline\";\nimport { spawn } from \"child_process\";\nimport { readFileSync } from \"fs\";\nimport path from \"path\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { isNativeSearchForced, nativeGrep } from \"../tools/native-search.js\";\nimport type { GrepLineHit } from \"./adapter.js\";\n\n/** Terms considered per query (longest first) when building the pattern. */\nconst MAX_TERMS = 4;\n/** Minimum token length worth matching on. */\nconst MIN_TERM_LENGTH = 3;\n\nfunction escapeRegExp(value: string): string {\n\treturn value.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n\nexport interface LexicalQueryPlan {\n\t/** rg-ready regex pattern. */\n\tpattern: string;\n\t/** Raw (unescaped, lowercased) terms, for per-line term attribution. */\n\tterms: string[];\n}\n\n/**\n * Build the retrieval plan for a query: a quoted segment is searched\n * verbatim; otherwise the longest few identifier-ish tokens are OR-ed\n * together. Returns undefined when the query yields nothing searchable.\n */\nexport function buildLexicalQueryPlan(query: string): LexicalQueryPlan | undefined {\n\tconst quoted = [...query.matchAll(/[\"'`]([^\"'`]+)[\"'`]/g)]\n\t\t.map((m) => m[1].trim())\n\t\t.filter((s) => s.length > 0)\n\t\t.sort((a, b) => b.length - a.length)[0];\n\tif (quoted) return { pattern: escapeRegExp(quoted), terms: [quoted.toLowerCase()] };\n\n\tconst tokens = [...new Set(query.match(/[A-Za-z0-9_$][\\w$.-]*/g) ?? [])]\n\t\t.filter((t) => t.length >= MIN_TERM_LENGTH)\n\t\t.sort((a, b) => b.length - a.length || a.localeCompare(b))\n\t\t.slice(0, MAX_TERMS);\n\tif (tokens.length === 0) {\n\t\tconst trimmed = query.trim();\n\t\treturn trimmed ? { pattern: escapeRegExp(trimmed), terms: [trimmed.toLowerCase()] } : undefined;\n\t}\n\treturn { pattern: tokens.map(escapeRegExp).join(\"|\"), terms: tokens.map((t) => t.toLowerCase()) };\n}\n\n/** Pattern-only view of {@link buildLexicalQueryPlan}. */\nexport function buildLexicalPattern(query: string): string | undefined {\n\treturn buildLexicalQueryPlan(query)?.pattern;\n}\n\n/** Which plan terms appear on a matched line (retrieval is case-insensitive,\n *  so attribution is too). */\nfunction termsOnLine(plan: LexicalQueryPlan, lineText: string | undefined): string[] {\n\tif (!lineText) return [];\n\tconst lower = lineText.toLowerCase();\n\treturn plan.terms.filter((t) => lower.includes(t));\n}\n\n/**\n * Run lexical retrieval over `cwd`, returning up to `limit` line-hits in\n * output order with POSIX repo-relative paths.\n */\nexport interface RunLexicalOptions {\n\tcwd: string;\n\tquery: string;\n\tlimit: number;\n\tglob?: string;\n\tsignal?: AbortSignal;\n\t/**\n\t * Restrict the search to these repo-relative files instead of walking `cwd`.\n\t *\n\t * Used to scope the grep leg to files the embedding index does not have\n\t * current content for. An empty array means \"no files to search\", which is\n\t * a real answer — not \"search everything\".\n\t */\n\tpaths?: readonly string[];\n}\n\n/**\n * Run the lexical retriever for the search tool. Optional glob filter scopes\n * file paths (slashless matches basename anywhere, slash patterns match the\n * full repo-relative path).\n */\nfunction normalizeSearchGlob(glob: string | undefined): string | undefined {\n\tif (!glob) return undefined;\n\t// Match fd/rg semantics: a slash-containing glob is anchored anywhere in\n\t// the tree, so prepend \"**/\" unless it already starts with a slash or \"**/\".\n\tif (glob.includes(\"/\") && !glob.startsWith(\"/\") && !glob.startsWith(\"**/\")) {\n\t\treturn `**/${glob}`;\n\t}\n\treturn glob;\n}\n\nexport async function runLexicalRetriever(options: RunLexicalOptions): Promise<GrepLineHit[]> {\n\tconst { cwd, query, limit, glob: rawGlob, signal, paths } = options;\n\tconst glob = normalizeSearchGlob(rawGlob);\n\tconst plan = buildLexicalQueryPlan(query);\n\tif (!plan) return [];\n\tconst { pattern } = plan;\n\t// Scoped to an empty set: nothing to search, and handing rg no paths would\n\t// make it read stdin and hang.\n\tif (paths?.length === 0) return [];\n\n\tconst toRel = (filePath: string): string => {\n\t\tconst rel = path.relative(cwd, filePath);\n\t\treturn (rel && !rel.startsWith(\"..\") ? rel : filePath).replace(/\\\\/g, \"/\");\n\t};\n\n\tconst rgPath = isNativeSearchForced() ? undefined : await ensureTool(\"rg\", true);\n\tif (!rgPath) {\n\t\t// nativeGrep takes one root, so a scoped run visits each file directly.\n\t\t// The scoped set is small by construction — it is what changed since the\n\t\t// last index pass — so this stays bounded.\n\t\tconst roots = paths ? paths.map((rel) => path.resolve(cwd, rel)) : [cwd];\n\t\tconst hits: GrepLineHit[] = [];\n\t\tfor (const root of roots) {\n\t\t\tif (hits.length >= limit) break;\n\t\t\tconst result = await nativeGrep(root, {\n\t\t\t\tpattern,\n\t\t\t\tisDirectory: paths === undefined,\n\t\t\t\tignoreCase: true,\n\t\t\t\tlimit: limit - hits.length,\n\t\t\t\tglob,\n\t\t\t\tsignal,\n\t\t\t\treadFile: (p) => readFileSync(p, \"utf-8\"),\n\t\t\t});\n\t\t\tfor (const m of result.matches) {\n\t\t\t\thits.push({ rel: toRel(m.filePath), line: m.lineNumber, terms: termsOnLine(plan, m.lineText) });\n\t\t\t}\n\t\t}\n\t\treturn hits;\n\t}\n\n\treturn new Promise<GrepLineHit[]>((resolve, reject) => {\n\t\t// --sort path forces a deterministic (single-threaded) walk: with the\n\t\t// match cap truncating the stream, a parallel walk would return a\n\t\t// different hit subset per run — \"same query, different context\".\n\t\t// `!.git` because --hidden would otherwise search .git contents;\n\t\t// --no-require-git so .gitignore is honored outside git repos too.\n\t\tconst args = [\n\t\t\t\"--json\",\n\t\t\t\"--line-number\",\n\t\t\t\"--color=never\",\n\t\t\t\"--hidden\",\n\t\t\t\"--no-require-git\",\n\t\t\t\"--ignore-case\",\n\t\t\t\"--sort\",\n\t\t\t\"path\",\n\t\t\t\"--glob\",\n\t\t\t\"!**/.git/**\",\n\t\t];\n\t\tif (glob) args.push(\"--glob\", glob);\n\t\targs.push(\"--\", pattern, ...(paths ? paths.map((rel) => path.resolve(cwd, rel)) : [cwd]));\n\t\tconst child = spawn(rgPath, args, { stdio: [\"ignore\", \"pipe\", \"pipe\"] });\n\t\tconst rl = createInterface({ input: child.stdout });\n\t\tconst hits: GrepLineHit[] = [];\n\t\tlet stderr = \"\";\n\t\tlet killedDueToLimit = false;\n\t\tlet aborted = false;\n\n\t\tconst onAbort = () => {\n\t\t\taborted = true;\n\t\t\tif (!child.killed) child.kill();\n\t\t};\n\t\tsignal?.addEventListener(\"abort\", onAbort, { once: true });\n\t\tchild.stderr?.on(\"data\", (chunk) => {\n\t\t\tstderr += chunk.toString();\n\t\t});\n\n\t\trl.on(\"line\", (line) => {\n\t\t\tif (!line.trim() || hits.length >= limit) return;\n\t\t\tlet event: any;\n\t\t\ttry {\n\t\t\t\tevent = JSON.parse(line);\n\t\t\t} catch {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tif (event.type !== \"match\") return;\n\t\t\tconst filePath = event.data?.path?.text;\n\t\t\tconst lineNumber = event.data?.line_number;\n\t\t\tif (filePath && typeof lineNumber === \"number\") {\n\t\t\t\thits.push({ rel: toRel(filePath), line: lineNumber, terms: termsOnLine(plan, event.data?.lines?.text) });\n\t\t\t}\n\t\t\tif (hits.length >= limit && !child.killed) {\n\t\t\t\tkilledDueToLimit = true;\n\t\t\t\tchild.kill();\n\t\t\t}\n\t\t});\n\n\t\tchild.on(\"error\", (error) => {\n\t\t\tsignal?.removeEventListener(\"abort\", onAbort);\n\t\t\treject(new Error(`Failed to run ripgrep: ${error.message}`));\n\t\t});\n\t\tchild.on(\"close\", (code) => {\n\t\t\trl.close();\n\t\t\tsignal?.removeEventListener(\"abort\", onAbort);\n\t\t\tif (aborted) {\n\t\t\t\treject(new Error(\"Operation aborted\"));\n\t\t\t\treturn;\n\t\t\t}\n\t\t\t// rg exits 1 on \"no matches\" — that is a valid empty result.\n\t\t\tif (!killedDueToLimit && code !== 0 && code !== 1) {\n\t\t\t\treject(new Error(stderr.trim() || `ripgrep exited with code ${code}`));\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tresolve(hits);\n\t\t});\n\t});\n}\n"]}