/** * FileIR emitter — runs tree-sitter queries against a parsed tree and * assembles the neutral intermediate representation. * * Determinism (rule 38): every collection is sorted before it leaves this * module. Sort keys are chosen so output is byte-identical across runs: * symbols → (startByte, name) * imports → (startByte, module) * exports → (startByte, name) * callSites → (startByte, firstCandidate) * routes → (startByte, pathTemplate) */ import { createHash } from "node:crypto"; import { Query, type Language, type Node as TSNode } from "web-tree-sitter"; import type { CallSiteIR, ExportIR, FileIR, ImportIR, RouteIR, SymbolIR, } from "@remnic/core/coding/coding-graph-types"; import type { CodingGraphLanguage } from "@remnic/core"; import { EXTRACTORS, kindFromCapture, type DefKind, } from "./extractors.js"; import { buildUtf16ToByteOffsetMap, utf16ToByte } from "./utf16-offsets.js"; // --------------------------------------------------------------------------- // Content hashing — SHA-256 of the raw bytes (rule 23). // --------------------------------------------------------------------------- export function hashContent(content: Uint8Array): string { return createHash("sha256").update(content).digest("hex"); } // --------------------------------------------------------------------------- // Module-specifier cleanup — strips quotes/brackets from the captured text. // --------------------------------------------------------------------------- function cleanModuleSpecifier(raw: string): string { let s = raw.trim(); // Strip C/C++ system includes: if (s.startsWith("<") && s.endsWith(">")) return s.slice(1, -1); // Strip double/single/backtick quotes. if (s.length >= 2) { const f = s[0]; const l = s[s.length - 1]; if ((f === '"' || f === "'" || f === "`") && f === l) return s.slice(1, -1); } return s; } // --------------------------------------------------------------------------- // Symbol extraction with qualified-name computation. // // Qualified names are computed via a nesting stack: definitions are sorted by // startByte; for each definition we pop the stack until the top contains the // current definition's start byte. The qualified name is the join of the // stack's names plus the current name. This correctly handles sibling methods // in a class, nested classes, etc. // --------------------------------------------------------------------------- interface RawDef { readonly kind: DefKind; readonly name: string; readonly startByte: number; readonly endByte: number; /** Go receiver type (e.g. "Server" from `func (s *Server) Start()`). */ readonly receiverType?: string; } function extractSymbols(root: TSNode, language: Language, lang: CodingGraphLanguage): SymbolIR[] { const extractor = EXTRACTORS[lang]; const query = new Query(language, extractor.definitionsQuery); try { const matches = query.matches(root); const rawDefs: RawDef[] = []; for (const match of matches) { let kind: DefKind | null = null; let nameNode: TSNode | null = null; let defNode: TSNode | null = null; let receiverType = ""; for (const cap of match.captures) { const k = kindFromCapture(cap.name); if (k) { kind = k; defNode = cap.node; } else if (cap.name === "name") { nameNode = cap.node; } else if (cap.name === "__receiver.type") { receiverType = cap.node.text; } } if (!kind || !defNode || !nameNode) continue; rawDefs.push({ kind, name: nameNode.text, startByte: defNode.startIndex, endByte: defNode.endIndex, receiverType: receiverType || undefined, }); } // Deduplicate: a function_item inside an impl block matches both the // general function_item pattern and the impl-scoped method pattern. // Keep the method version (which carries the receiver type for parent // qualification). Same startByte+endByte+name guarantees it's the same // AST node matched by two query patterns, not two distinct definitions. const seen = new Map(); for (const def of rawDefs) { const key = `${def.startByte}:${def.endByte}:${def.name}`; const existing = seen.get(key); if (!existing || (def.receiverType && !existing.receiverType)) { seen.set(key, def); } } const deduped = [...seen.values()]; deduped.sort((a, b) => a.startByte - b.startByte || a.name.localeCompare(b.name)); // The stack stores each def's computed qualifiedName so that nested // definitions inside a receiver-qualified method (e.g. Config.new.helper) // get the full parent chain, not just the short method name. const stack: { endByte: number; qualifiedName: string }[] = []; const symbols: SymbolIR[] = []; for (const def of deduped) { while (stack.length > 0 && stack[stack.length - 1].endByte <= def.startByte) { stack.pop(); } // Go/Rust methods sit outside their receiver struct, so byte-span // nesting cannot determine the parent. Use the captured receiver // type instead. // The last stack entry's qualifiedName already contains the full // ancestor chain (e.g. "Server.start"), so use it directly rather // than joining all entries (which would duplicate ancestors). const parentQualifiedName = def.receiverType ?? (stack.length > 0 ? stack[stack.length - 1].qualifiedName : undefined); const qualifiedName = parentQualifiedName ? `${parentQualifiedName}.${def.name}` : def.name; const symbol: SymbolIR = parentQualifiedName ? { kind: def.kind, name: def.name, qualifiedName, span: { startByte: def.startByte, endByte: def.endByte }, parentQualifiedName, } : { kind: def.kind, name: def.name, qualifiedName, span: { startByte: def.startByte, endByte: def.endByte }, }; symbols.push(symbol); stack.push({ endByte: def.endByte, qualifiedName }); } return symbols; } finally { query.delete(); } } // --------------------------------------------------------------------------- // Import extraction — group captures by @__import.stmt node. // --------------------------------------------------------------------------- function extractImports(root: TSNode, language: Language, lang: CodingGraphLanguage): ImportIR[] { const extractor = EXTRACTORS[lang]; if (!extractor.importsQuery) return []; const query = new Query(language, extractor.importsQuery); try { const matches = query.matches(root); // Group by import-statement node start index (unique per node in tree). // Group by (statement-start + module) so multi-module statements like // Python `import os, sys` produce separate import entries rather than // collapsing to a single module. Single-module statements like // `import { foo, bar } from "module"` still group correctly because // all captures share the same module. const groups = new Map< string, { module: string; names: Set; bindings: Map; // local -> exported (issue #1894 review) startByte: number; endByte: number; } >(); for (const match of matches) { let moduleText = ""; let stmtStart = -1; let stmtEnd = -1; const names: string[] = []; // Default/namespace import locals: informational only, never // call-bindable (issue #1894 round 12). const unboundNames: string[] = []; // local -> exported for aliased named imports (issue #1894 review). let aliasExported = ""; let aliasLocal = ""; for (const cap of match.captures) { if (cap.name === "import.module") { moduleText = cleanModuleSpecifier(cap.node.text); } else if (cap.name === "import.name") { names.push(cap.node.text); } else if (cap.name === "import.unboundName") { unboundNames.push(cap.node.text); } else if (cap.name === "import.aliasExported") { aliasExported = cap.node.text; } else if (cap.name === "import.aliasLocal") { aliasLocal = cap.node.text; } else if (cap.name === "__import.stmt") { stmtStart = cap.node.startIndex; stmtEnd = cap.node.endIndex; } } if (stmtStart < 0) { // Fallback: use the first capture's parent chain to find an import node. const firstCap = match.captures[0]; if (firstCap) { stmtStart = firstCap.node.startIndex; stmtEnd = firstCap.node.endIndex; } else { continue; } } const key = `${stmtStart}:${moduleText}`; let group = groups.get(key); if (!group) { group = { module: moduleText, names: new Set(), bindings: new Map(), startByte: stmtStart, endByte: stmtEnd, }; groups.set(key, group); } for (const n of names) { group.names.add(n); // Non-aliased: local === exported. May be overwritten below when // the alias pattern also matched this specifier. if (!group.bindings.has(n)) group.bindings.set(n, n); } for (const n of unboundNames) { group.names.add(n); } if (aliasExported && aliasLocal) { group.names.add(aliasExported); // The plain-name pattern also matched this specifier and recorded // exported->exported; the alias binding replaces it: the LOCAL // identifier is what call sites use. group.bindings.delete(aliasExported); group.bindings.set(aliasLocal, aliasExported); } } return Array.from(groups.values()) .map((g) => ({ module: g.module, importedNames: Array.from(g.names).sort(), bindings: Array.from(g.bindings.entries()) .map(([local, exported]) => ({ exported, local })) .sort((a, b) => a.local.localeCompare(b.local)), span: { startByte: g.startByte, endByte: g.endByte }, })) .sort((a, b) => a.span.startByte - b.span.startByte || a.module.localeCompare(b.module)); } finally { query.delete(); } } // --------------------------------------------------------------------------- // Export extraction. // --------------------------------------------------------------------------- function extractExports(root: TSNode, language: Language, lang: CodingGraphLanguage): ExportIR[] { const extractor = EXTRACTORS[lang]; if (!extractor.exportsQuery) return []; const query = new Query(language, extractor.exportsQuery); try { const captures = query.captures(root); // Dedup a CommonJS pair overlap (#1659 review): a pair // `{ key: value }` whose value is an identifier is matched by BOTH // the value-identifier pattern (captures the real symbol) AND the // non-identifier fallback (captures the key). The fallback's // #not-match? regex is ASCII-only, so a Unicode identifier value // (e.g. Universität) defeats it and both patterns fire on the same // pair, duplicating the export. web-tree-sitter's query regex // engine does not support \p{L}, so dedup here: if a pair already // exported its value identifier, drop the spurious key capture. const valueExportedPairs = new Set(); const pairOf = (node: TSNode): TSNode | null => { let cur: TSNode | null = node; for (let i = 0; i < 5 && cur; i++) { if (cur.type === "pair") return cur; cur = cur.parent; } return null; }; for (const cap of captures) { if (cap.name !== "export.name") continue; const pair = pairOf(cap.node); if (pair && cap.node.type === "identifier") { valueExportedPairs.add(pair.id); } } const exports: ExportIR[] = []; for (const cap of captures) { if (cap.name !== "export.name") continue; const pair = pairOf(cap.node); if ( pair && cap.node.type === "property_identifier" && valueExportedPairs.has(pair.id) ) { continue; // value identifier is the real export; drop the alias key } exports.push({ name: cap.node.text, span: { startByte: cap.node.startIndex, endByte: cap.node.endIndex }, }); } return exports.sort( (a, b) => a.span.startByte - b.span.startByte || a.name.localeCompare(b.name), ); } finally { query.delete(); } } // --------------------------------------------------------------------------- // Call-site extraction. // --------------------------------------------------------------------------- function extractCallSites(root: TSNode, language: Language, lang: CodingGraphLanguage): CallSiteIR[] { const extractor = EXTRACTORS[lang]; if (!extractor.callSitesQuery) return []; const query = new Query(language, extractor.callSitesQuery); try { const captures = query.captures(root); const callSites: CallSiteIR[] = []; for (const cap of captures) { if (cap.name === "call.callee") { callSites.push({ calleeNameCandidates: [cap.node.text], span: { startByte: cap.node.startIndex, endByte: cap.node.endIndex }, }); } else if (cap.name === "call.member") { // Member/property call (issue #1894 review): the bare property // name must never bind a visible symbol — mark it so the // heuristic resolver skips it (LSP owns method dispatch). callSites.push({ calleeNameCandidates: [cap.node.text], memberAccess: true, span: { startByte: cap.node.startIndex, endByte: cap.node.endIndex }, }); } } return callSites.sort( (a, b) => a.span.startByte - b.span.startByte || (a.calleeNameCandidates[0] ?? "").localeCompare(b.calleeNameCandidates[0] ?? ""), ); } finally { query.delete(); } } // --------------------------------------------------------------------------- // Route extraction (Express/Fastify/Flask/etc.). // --------------------------------------------------------------------------- // Common HTTP client variable names that should NOT produce routes. // These objects have methods named get/post/etc. that match the route // verb pattern but are client-side calls, not server route registrations. // Without this exclusion, httpClient.get("/api", opts, cb) would produce // a spurious route with handler=cb, marking cb as is_route_handler and // hiding it from dead-code detection (chatgpt-codex-connector #1688 P2). const HTTP_CLIENT_OBJECT_PATTERNS = /^(http|https|client|httpClient|axios|fetch|request|req|res|\$|superagent|got)$/; function extractRoutes(root: TSNode, language: Language, lang: CodingGraphLanguage): RouteIR[] { const extractor = EXTRACTORS[lang]; if (!extractor.routesQuery) return []; const query = new Query(language, extractor.routesQuery); try { const matches = query.matches(root); const routes: RouteIR[] = []; for (const match of matches) { let verb = ""; let pathTemplate = ""; let handler = ""; let startByte = 0; let endByte = 0; let argsNode: TSNode | null = null; let routeObject = ""; for (const cap of match.captures) { if (cap.name === "route.verb") { verb = cap.node.text.toUpperCase(); startByte = cap.node.parent?.startIndex ?? cap.node.startIndex; // Extract the receiver object name for the HTTP-client exclusion. const memberExpr = cap.node.parent; const objectNode = memberExpr?.childForFieldName("object"); if (objectNode) { // Normalize nested receivers to their tail property so a call // like this.client.get("/api", opts, cb) is caught by the HTTP- // client exclusion. objectNode.text for `this.client` is // "this.client", which misses the ^client$ pattern; descend to // the rightmost property (chatgpt-codex-connector #1688 P2: // 'Normalize receiver names before client-route filtering'). let receiver = objectNode; for ( let prop = receiver.childForFieldName("property"); prop; prop = receiver.childForFieldName("property") ) { receiver = prop; } routeObject = receiver.text; } } else if (cap.name === "route.path") { pathTemplate = cleanModuleSpecifier(cap.node.text); } else if (cap.name === "route.handler") { // Python route handlers (function names) and legacy JS patterns. handler = cap.node.type === "identifier" ? cap.node.text : (findHandlerName(cap.node) ?? "anonymous"); endByte = cap.node.endIndex; } else if (cap.name === "route.args") { argsNode = cap.node; endByte = cap.node.endIndex; } } // Extract handler from the last argument when we captured the args // node (JS routes). Handles middleware: handler is the LAST arg (#1659 #5). if (argsNode) { handler = extractHandlerFromArgs(argsNode); } // Guards: (1) path-prefix — routes start with "/" or "*"; // (2) HTTP-client exclusion — objects named http/client/axios/etc. // are clients, not routers. Together these filter the most common // non-route call expressions that match the verb+string-arg pattern // (chatgpt-codex-connector #1688 P2: 'Reject client callbacks'). const isRoutePath = pathTemplate.startsWith("/") || pathTemplate.startsWith("*"); const isHttpClient = HTTP_CLIENT_OBJECT_PATTERNS.test(routeObject); if (verb && pathTemplate && handler && isRoutePath && !isHttpClient) { routes.push({ verb, pathTemplate, handlerQualifiedName: handler, span: { startByte, endByte }, }); } } return routes.sort( (a, b) => a.span.startByte - b.span.startByte || a.pathTemplate.localeCompare(b.pathTemplate), ); } finally { query.delete(); } } /** * Extract the route handler name from the LAST argument of an arguments * node. Handles middleware: app.get("/path", requireAuth, getUsers) → * handler=getUsers (the last arg), not requireAuth (issue #1659 #5). */ function extractHandlerFromArgs(argsNode: TSNode): string { // Collect the real (non-comment) named args, skipping trailing inline/ // block comments. tree-sitter treats comments as named children, so // `app.get("/users", getUsers /* auth */)` would otherwise select the // comment as the last arg, miss the real handler, and leave it // un-protected by the route-handler exclusion (a false dead-code hit). // (chatgpt-codex-connector #1659 review: 'Skip comments when selecting // route handler'.) const realArgs: TSNode[] = []; for (let i = 0; i < argsNode.namedChildCount; i++) { const child = argsNode.namedChild(i); if (child && child.type !== "comment") realArgs.push(child); } if (realArgs.length < 2) return ""; const lastArg = realArgs[realArgs.length - 1]!; if (lastArg.type === "identifier") { return lastArg.text; } if (lastArg.type === "function_expression") { return findHandlerName(lastArg) ?? "anonymous"; } if (lastArg.type === "arrow_function") { return "anonymous"; } // Non-handler last arg (object, number, call expression, etc.) — // not a route handler. Return empty so the caller skips the route // (cursor Bugbot: 'Spurious routes from client calls'). return ""; } /** * Try to find a handler name from a function/arrow expression node. * For named function expressions: `(function foo() {})` → "foo". * For arrow functions assigned to a variable, the variable name is not in this node; * the caller would need the parent. For now, return "anonymous" unless we find a name. */ function findHandlerName(node: TSNode): string | null { // function_expression may have a name child (identifier) for (const child of node.namedChildren) { if (child && child.type === "identifier") return child.text; } // Python function_definition has a name field const nameChild = node.childForFieldName("name"); if (nameChild) return nameChild.text; return null; } // --------------------------------------------------------------------------- // Top-level emitter. // --------------------------------------------------------------------------- /** * Assemble a FileIR from a parsed tree. All collections are sorted for * deterministic output (rule 38). * * `contentStr` is the UTF-8 string that was passed to the parser. It is used * to build a UTF-16→byte offset map so all spans are converted from UTF-16 * code-unit offsets (what web-tree-sitter returns) to UTF-8 byte offsets * (what on-disk files use). For ASCII-only content the two are identical; * multibyte content (comments, strings, identifiers) needs the conversion * (issue #1659 item 3). */ export function emitFileIR( filePath: string, lang: CodingGraphLanguage, content: Uint8Array, root: TSNode, language: Language, contentStr: string, ): FileIR { const symbols = extractSymbols(root, language, lang); const imports = extractImports(root, language, lang); const exports = extractExports(root, language, lang); const callSites = extractCallSites(root, language, lang); const routes = extractRoutes(root, language, lang); // Convert UTF-16 code-unit offsets → UTF-8 byte offsets (issue #1659 #3). // Spans are readonly, so rebuild each object with converted offsets. const offsetMap = buildUtf16ToByteOffsetMap(contentStr); const convSymbols = symbols.map((s) => ({ ...s, span: { startByte: utf16ToByte(offsetMap, s.span.startByte), endByte: utf16ToByte(offsetMap, s.span.endByte), }, })); const convImports = imports.map((i) => ({ ...i, span: { startByte: utf16ToByte(offsetMap, i.span.startByte), endByte: utf16ToByte(offsetMap, i.span.endByte), }, })); const convExports = exports.map((e) => ({ ...e, span: { startByte: utf16ToByte(offsetMap, e.span.startByte), endByte: utf16ToByte(offsetMap, e.span.endByte), }, })); const convCallSites = callSites.map((c) => ({ ...c, span: { startByte: utf16ToByte(offsetMap, c.span.startByte), endByte: utf16ToByte(offsetMap, c.span.endByte), }, })); const convRoutes = routes.map((r) => ({ ...r, span: { startByte: utf16ToByte(offsetMap, r.span.startByte), endByte: utf16ToByte(offsetMap, r.span.endByte), }, })); return { path: filePath, language: lang, contentHash: hashContent(content), symbols: convSymbols, imports: convImports, exports: convExports, callSites: convCallSites, routes: convRoutes, }; }