{"version":3,"file":"lexical.d.ts","sourceRoot":"","sources":["../../../src/core/workspace/lexical.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGhD,MAAM,WAAW,UAAU;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,aAAa,CAAC,EAAE,OAAO,CAAC;CACxB;AAYD,6CAA6C;AAC7C,wBAAgB,YAAY,CAC3B,EAAE,EAAE,WAAW,EACf,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,GACb,IAAI,CASN;AAED,wBAAgB,aAAa,CAAC,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAI1F;AAUD,4FAA4F;AAC5F,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAKzG;AAED,wBAAgB,aAAa,CAAC,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,UAAU,EAAE,CAsFrG;AAED,+EAA+E;AAC/E,wBAAgB,YAAY,CAAC,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,UAAU,EAAE,CAc/G;AAED,wEAAwE;AACxE,wBAAgB,gBAAgB,CAC/B,EAAE,EAAE,WAAW,EACf,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,EACZ,KAAK,SAAK,GACR,KAAK,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAsBpH","sourcesContent":["/**\n * Deterministic lexical retrieval.\n *\n * A portable postings-table + Okapi BM25 implementation over SQLite. This avoids\n * reliance on FTS5 availability (which can differ by Node build/platform) while\n * providing exact, prefix, phrase and identifier-aware full-text search with\n * deterministic ranking. All query text is parameter-bound, so no injection is\n * possible.\n */\n\nimport type { WorkspaceDb } from \"./storage.js\";\nimport { tokenizeQuery } from \"./tokenize.js\";\n\nexport interface LexicalHit {\n\tchunkId: string;\n\tpath: string;\n\tscore: number;\n\tmatchedTokens: string[];\n}\n\nexport interface LexicalQuery {\n\tquery: string;\n\tlimit?: number;\n\tfileClassFilters?: string[];\n\tlanguageFilters?: string[];\n\tpathFilter?: string;\n\tprefix?: boolean;\n\tcaseSensitive?: boolean;\n}\n\nconst K1 = 1.2;\nconst B = 0.75;\n\ninterface Candidate {\n\tchunkId: string;\n\tpath: string;\n\ttermFreq: Map<string, number>;\n\tdocLength: number;\n}\n\n/** Write the lexical records for a chunk. */\nexport function writeLexical(\n\tdb: WorkspaceDb,\n\tgenerationId: string,\n\tchunkId: string,\n\tpath: string,\n\tlexText: string,\n): void {\n\tconst tokens = tokenizeQuery(lexText);\n\tconst stmt = db.db.prepare(\n\t\t\"INSERT OR REPLACE INTO lex (generation_id, chunk_id, path, doc_tokens, doc_length) VALUES (?, ?, ?, ?, ?)\",\n\t);\n\tstmt.run(generationId, chunkId, path, tokens.join(\" \"), tokens.length);\n\tconst upsert = db.db.prepare(\"INSERT OR IGNORE INTO postings (generation_id, token, chunk_id) VALUES (?, ?, ?)\");\n\t// Deduplicate tokens per chunk for the postings table.\n\tfor (const token of new Set(tokens)) upsert.run(generationId, token, chunkId);\n}\n\nexport function deleteLexical(db: WorkspaceDb, generationId: string, chunkId: string): void {\n\tdb.db.prepare(\"DELETE FROM lex WHERE generation_id = ? AND chunk_id = ?\").run(generationId, chunkId);\n\tdb.db.prepare(\"DELETE FROM postings WHERE generation_id = ? AND chunk_id = ?\").run(generationId, chunkId);\n\tdb.db.prepare(\"DELETE FROM chunks_fts WHERE chunk_id = ?\").run(chunkId);\n}\n\nfunction docPaths(db: WorkspaceDb, generationId: string): Map<string, string> {\n\tconst rows = db.db.prepare(\"SELECT chunk_id, path FROM lex WHERE generation_id = ?\").all(generationId) as Array<{\n\t\tchunk_id: string;\n\t\tpath: string;\n\t}>;\n\treturn new Map(rows.map((r) => [r.chunk_id, r.path]));\n}\n\n/** Kill-list chunk_ids from a previous fully-indexed generation to remove stale vectors. */\nexport function chunkIdsForRemoval(db: WorkspaceDb, generationId: string, retained: Set<string>): string[] {\n\tconst rows = db.db.prepare(\"SELECT chunk_id FROM lex WHERE generation_id = ?\").all(generationId) as Array<{\n\t\tchunk_id: string;\n\t}>;\n\treturn rows.map((r) => r.chunk_id).filter((id) => !retained.has(id));\n}\n\nexport function searchLexical(db: WorkspaceDb, generationId: string, opts: LexicalQuery): LexicalHit[] {\n\tconst limit = Math.max(1, opts.limit ?? 50);\n\tconst rawTokens = tokenizeQuery(opts.query);\n\tif (rawTokens.length === 0) return [];\n\n\t// Prefix expansion: match tokens that start with the query token when requested.\n\tlet tokens = rawTokens;\n\tif (opts.prefix) {\n\t\tconst expanded = new Set<string>();\n\t\tfor (const t of rawTokens) {\n\t\t\texpanded.add(t);\n\t\t\tconst rows = db.db\n\t\t\t\t.prepare(\"SELECT DISTINCT token FROM postings WHERE generation_id = ? AND token LIKE ? LIMIT 200\")\n\t\t\t\t.all(generationId, `${t}%`) as Array<{ token: string }>;\n\t\t\tfor (const r of rows) expanded.add(r.token);\n\t\t}\n\t\ttokens = [...expanded];\n\t}\n\n\t// Candidate chunk ids from postings (bounded).\n\tconst candidateSet = new Map<string, Set<string>>(); // token -> chunkIds\n\tfor (const token of tokens) {\n\t\tconst rows = db.db\n\t\t\t.prepare(\"SELECT chunk_id FROM postings WHERE generation_id = ? AND token = ? LIMIT 5000\")\n\t\t\t.all(generationId, token) as Array<{ chunk_id: string }>;\n\t\tcandidateSet.set(token, new Set(rows.map((r) => r.chunk_id)));\n\t}\n\t// Chunks that contain at least one query token.\n\tconst union = new Set<string>();\n\tfor (const set of candidateSet.values()) for (const id of set) union.add(id);\n\tif (union.size === 0) return [];\n\n\t// Load doc tokens for candidates.\n\tconst chunkIds = [...union];\n\tconst placeholders = chunkIds.map(() => \"?\").join(\",\");\n\tconst lexRows = db.db\n\t\t.prepare(\n\t\t\t`SELECT chunk_id, doc_tokens, doc_length FROM lex WHERE generation_id = ? AND chunk_id IN (${placeholders})`,\n\t\t)\n\t\t.all(generationId, ...chunkIds) as Array<{ chunk_id: string; doc_tokens: string; doc_length: number }>;\n\n\tconst pathByChunk = docPaths(db, generationId);\n\n\tconst docs = new Map<string, Candidate>();\n\tfor (const r of lexRows) {\n\t\tconst freq = new Map<string, number>();\n\t\tfor (const t of r.doc_tokens.split(\" \")) freq.set(t, (freq.get(t) ?? 0) + 1);\n\t\tdocs.set(r.chunk_id, {\n\t\t\tchunkId: r.chunk_id,\n\t\t\tpath: pathByChunk.get(r.chunk_id) ?? \"\",\n\t\t\ttermFreq: freq,\n\t\t\tdocLength: r.doc_length,\n\t\t});\n\t}\n\n\tconst avgdl = docs.size ? [...docs.values()].reduce((a, d) => a + d.docLength, 0) / docs.size : 1;\n\tconst totalDocs = db.db.prepare(\"SELECT COUNT(*) c FROM lex WHERE generation_id = ?\").get(generationId) as {\n\t\tc: number;\n\t};\n\n\t// IDT (inverse document token) per token.\n\tconst idf = new Map<string, number>();\n\tfor (const token of tokens) {\n\t\tconst df = candidateSet.get(token)?.size ?? 0;\n\t\tconst n = Math.max(1, totalDocs.c);\n\t\tidf.set(token, Math.log(1 + (n - df + 0.5) / (df + 0.5)));\n\t}\n\n\tconst hits: LexicalHit[] = [];\n\tfor (const doc of docs.values()) {\n\t\tlet score = 0;\n\t\tconst matched: string[] = [];\n\t\tfor (const token of tokens) {\n\t\t\tconst tf = doc.termFreq.get(token) ?? 0;\n\t\t\tif (tf === 0) continue;\n\t\t\tmatched.push(token);\n\t\t\tconst denom = tf + K1 * (1 - B + (B * doc.docLength) / avgdl);\n\t\t\tscore += (idf.get(token) ?? 0) * ((tf * (K1 + 1)) / denom);\n\t\t}\n\t\tif (matched.length > 0) {\n\t\t\thits.push({ chunkId: doc.chunkId, path: doc.path, score, matchedTokens: matched });\n\t\t}\n\t}\n\n\thits.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path) || a.chunkId.localeCompare(b.chunkId));\n\treturn hits.slice(0, limit);\n}\n\n/** Path-prefix search: locate chunks under a given workspace-relative path. */\nexport function searchByPath(db: WorkspaceDb, generationId: string, pathQuery: string, limit = 50): LexicalHit[] {\n\tconst q = pathQuery.replace(/\\\\/g, \"/\").toLowerCase().replace(/^\\/+/, \"\");\n\tconst rows = db.db.prepare(\"SELECT chunk_id, path FROM lex WHERE generation_id = ?\").all(generationId) as Array<{\n\t\tchunk_id: string;\n\t\tpath: string;\n\t}>;\n\tconst hits: LexicalHit[] = [];\n\tfor (const r of rows) {\n\t\tif (r.path.toLowerCase().includes(q)) {\n\t\t\thits.push({ chunkId: r.chunk_id, path: r.path, score: 1 / (1 + r.path.length), matchedTokens: [] });\n\t\t}\n\t}\n\thits.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));\n\treturn hits.slice(0, limit);\n}\n\n/** Identifier-aware exact symbol-name search over the symbols table. */\nexport function searchSymbolName(\n\tdb: WorkspaceDb,\n\tgenerationId: string,\n\tname: string,\n\tlimit = 50,\n): Array<{ symbolId: string; fileId: string; name: string; qualifiedName?: string; kind: string; startLine: number }> {\n\tconst q = name.toLowerCase();\n\tconst rows = db.db\n\t\t.prepare(\n\t\t\t\"SELECT symbol_id, file_id, name, qualified_name, kind, start_line FROM symbols WHERE generation_id = ? AND lower(name) = ? LIMIT ?\",\n\t\t)\n\t\t.all(generationId, q, limit) as Array<{\n\t\tsymbol_id: string;\n\t\tfile_id: string;\n\t\tname: string;\n\t\tqualified_name: string | null;\n\t\tkind: string;\n\t\tstart_line: number;\n\t}>;\n\treturn rows.map((r) => ({\n\t\tsymbolId: r.symbol_id,\n\t\tfileId: r.file_id,\n\t\tname: r.name,\n\t\tqualifiedName: r.qualified_name ?? undefined,\n\t\tkind: r.kind,\n\t\tstartLine: r.start_line,\n\t}));\n}\n"]}