{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../../src/core/workspace/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAgBH,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,2DAA2D;AAC3D,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAUrD;AAED,mEAAmE;AACnE,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,mBAAmB,CAM1E;AAED,sEAAsE;AACtE,wBAAgB,gBAAgB,CAAC,OAAO,EAAE;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,MAAM,CAOT","sourcesContent":["/**\n * Identifier-aware tokenization for code.\n *\n * Splits identifiers on camelCase, snake_case, kebab-case, and separators while\n * preserving the exact original token. Produces both the original token and its\n * normalized search tokens.\n */\n\nconst SPLIT_RE = /[^A-Za-z0-9_]+/g;\n\nfunction camelSplit(word: string): string[] {\n\tconst tokens: string[] = [];\n\t// Split on case transitions / digits boundaries and underscores.\n\tconst parts = word\n\t\t.replace(/([a-z0-9])([A-Z])/g, \"$1\\u0000$2\")\n\t\t.replace(/([A-Z]+)([A-Z][a-z])/g, \"$1\\u0000$2\")\n\t\t.split(/[\\u0000_]+/)\n\t\t.filter(Boolean);\n\tfor (const p of parts) tokens.push(p.toLowerCase());\n\treturn tokens;\n}\n\nexport interface TokenizedIdentifier {\n\toriginal: string;\n\tlower: string;\n\tsearchTokens: string[];\n}\n\n/** Tokenize a full query into normalized search tokens. */\nexport function tokenizeQuery(query: string): string[] {\n\tconst tokens = new Set<string>();\n\tconst cleaned = query.replace(SPLIT_RE, \" \");\n\tfor (const word of cleaned.trim().split(/\\s+/)) {\n\t\tif (!word) continue;\n\t\tfor (const t of camelSplit(word)) if (t.length >= 1) tokens.add(t);\n\t\tconst lowered = word.toLowerCase();\n\t\tif (lowered.length >= 2) tokens.add(lowered);\n\t}\n\treturn [...tokens];\n}\n\n/** Tokenize a single code identifier (preserving the original). */\nexport function tokenizeIdentifier(identifier: string): TokenizedIdentifier {\n\treturn {\n\t\toriginal: identifier,\n\t\tlower: identifier.toLowerCase(),\n\t\tsearchTokens: camelSplit(identifier),\n\t};\n}\n\n/** Produce the document text used for lexical indexing of a chunk. */\nexport function lexicalChunkText(options: {\n\tpath: string;\n\ttext: string;\n\tsymbolName?: string;\n\tqualifiedName?: string;\n}): string {\n\tconst pathTokens = tokenizeQuery(options.path.replace(/[./\\\\]+/g, \" \"));\n\tconst nameTokens = options.symbolName\n\t\t? [...tokenizeIdentifier(options.symbolName).searchTokens, options.symbolName.toLowerCase()]\n\t\t: [];\n\tconst qTokens = options.qualifiedName ? tokenizeIdentifier(options.qualifiedName).searchTokens : [];\n\treturn [...pathTokens, ...nameTokens, ...qTokens, options.text].join(\" \");\n}\n"]}