import { readdir, readFile, stat } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { extname, join, normalize, relative, resolve, sep } from 'node:path'; import type { KnowledgeGraph, KnowledgeGraphEdge, KnowledgeGraphNode } from '../index.js'; import { EXCLUDED_DIRS } from './builder.js'; const cjsRequire = createRequire(__filename); type TreeSitterParser = { setLanguage: (language: unknown) => void; parse: (source: string) => TreeSitterTree; }; type TreeSitterNode = { type: string; startIndex: number; endIndex: number; namedChildren: TreeSitterNode[]; parent: TreeSitterNode | null; childForFieldName: (name: string) => TreeSitterNode | null; }; type TreeSitterTree = { rootNode: TreeSitterNode; }; type TreeSitterParserCtor = new () => TreeSitterParser; type ParsedFileData = { imports: string[]; functions: string[]; }; type ParserRuntime = { Parser: TreeSitterParserCtor; Python: unknown; TypeScript: unknown; TSX: unknown; JavaScript: unknown; }; const MAX_FILE_SIZE_BYTES = 500 * 1024; const BATCH_SIZE = 10; const SUPPORTED_EXTENSIONS = new Set(['.py', '.ts', '.tsx', '.js', '.jsx']); const EDGE_TYPE_IMPORTS = 'imports'; const BUILTIN_MODULES = new Set([ 'assert', 'buffer', 'child_process', 'cluster', 'console', 'constants', 'crypto', 'dgram', 'dns', 'events', 'fs', 'http', 'http2', 'https', 'module', 'net', 'os', 'path', 'perf_hooks', 'process', 'querystring', 'readline', 'stream', 'string_decoder', 'sys', 'timers', 'tls', 'tty', 'url', 'util', 'v8', 'vm', 'worker_threads', 'zlib', ]); export async function analyzeProjectWithTreeSitter( projectPath: string, onProgress?: (message: string) => void, ): Promise { const runtime = loadTreeSitterRuntime(); const rootPath = resolve(projectPath); const files = await collectCandidateFiles(rootPath, onProgress); const fileSet = new Set(files); const parsedByFile = new Map(); onProgress?.(`🌳 Tree-sitter: parsing ${String(files.length)} files...`); for (let index = 0; index < files.length; index += BATCH_SIZE) { const batch = files.slice(index, index + BATCH_SIZE); await Promise.all( batch.map(async (filePath) => { const parsed = await parseSingleFile(rootPath, filePath, runtime, onProgress); if (parsed !== null) { parsedByFile.set(filePath, parsed); } }), ); } const nodes: KnowledgeGraphNode[] = []; for (const filePath of files) { const parsed = parsedByFile.get(filePath); nodes.push({ id: filePath, type: 'file', name: filePath.split('/').at(-1) ?? filePath, path: filePath, layer: inferLayer(filePath), summary: '', functions: parsed?.functions ?? [], imports: parsed?.imports ?? [], }); } const edges = resolveImportEdges(rootPath, parsedByFile, fileSet); return { nodes, edges }; } function loadTreeSitterRuntime(): ParserRuntime { const Parser = cjsRequire('tree-sitter') as TreeSitterParserCtor; const Python = cjsRequire('tree-sitter-python') as unknown; const tsModule = cjsRequire('tree-sitter-typescript') as { typescript: unknown; tsx: unknown }; const JavaScript = cjsRequire('tree-sitter-javascript') as unknown; return { Parser, Python, TypeScript: tsModule.typescript, TSX: tsModule.tsx, JavaScript, }; } function getGrammar(extension: string, runtime: ParserRuntime): unknown | null { switch (extension) { case '.py': return runtime.Python; case '.ts': return runtime.TypeScript; case '.tsx': return runtime.TSX; case '.js': case '.jsx': return runtime.JavaScript; default: return null; } } async function collectCandidateFiles( projectPath: string, onProgress?: (message: string) => void, ): Promise { const collected: string[] = []; await walk(projectPath); return collected; async function walk(currentPath: string): Promise { const entries = await readdir(currentPath, { withFileTypes: true }); for (const entry of entries) { const fullPath = join(currentPath, entry.name); if (entry.isDirectory()) { if (EXCLUDED_DIRS.has(entry.name)) { continue; } if (isExplicitlyExcludedDir(fullPath, projectPath)) { continue; } await walk(fullPath); continue; } if (!entry.isFile()) { continue; } const extension = extname(entry.name).toLowerCase(); if (!SUPPORTED_EXTENSIONS.has(extension)) { continue; } if (isExplicitlyExcludedFile(entry.name)) { continue; } const fileStats = await stat(fullPath); if (fileStats.size > MAX_FILE_SIZE_BYTES) { logWarning(onProgress, `Skipping large file >500KB: ${toPosix(relative(projectPath, fullPath))}`); continue; } collected.push(toPosix(relative(projectPath, fullPath))); } } } async function parseSingleFile( projectPath: string, relativeFilePath: string, runtime: ParserRuntime, onProgress?: (message: string) => void, ): Promise { const extension = extname(relativeFilePath).toLowerCase(); const grammar = getGrammar(extension, runtime); if (grammar === null) { return null; } const parser = new runtime.Parser(); parser.setLanguage(grammar); const absolutePath = resolve(projectPath, relativeFilePath); let source = ''; try { source = await readFile(absolutePath, 'utf8'); } catch (error) { logWarning(onProgress, `Failed to read ${relativeFilePath}: ${errorToMessage(error)}`); return null; } try { const tree = parser.parse(source); const imports: string[] = []; const functions = new Set(); traverseNode(tree.rootNode, (node) => { switch (extension) { case '.py': collectPythonData(node, source, imports, functions, relativeFilePath); break; case '.ts': case '.tsx': case '.js': case '.jsx': collectTsJsData(node, source, imports, functions); break; default: break; } }); return { imports, functions: [...functions], }; } catch (error) { logWarning(onProgress, `Failed to parse ${relativeFilePath}: ${errorToMessage(error)}`); return null; } } function collectPythonData( node: TreeSitterNode, source: string, imports: string[], functions: Set, relativeFilePath: string, ): void { if (node.type === 'import_statement') { const text = nodeText(source, node); const modules = extractPythonImportModules(text); imports.push(...modules); return; } if (node.type === 'import_from_statement') { const text = nodeText(source, node); const modulePath = extractPythonFromModule(text, relativeFilePath); if (modulePath !== null) { imports.push(modulePath); } return; } if (node.type === 'function_definition') { const nameNode = node.childForFieldName('name'); if (nameNode !== null) { functions.add(nodeText(source, nameNode)); } return; } if (node.type === 'class_definition') { const nameNode = node.childForFieldName('name'); if (nameNode !== null) { functions.add(`class:${nodeText(source, nameNode)}`); } } } function collectTsJsData( node: TreeSitterNode, source: string, imports: string[], functions: Set, ): void { if (node.type === 'import_declaration') { const text = nodeText(source, node); const match = /from\s+['\"]([^'\"]+)['\"]/.exec(text); if (match !== null) { imports.push(match[1]); } return; } if (node.type === 'function_declaration') { const nameNode = node.childForFieldName('name'); if (nameNode !== null) { functions.add(nodeText(source, nameNode)); } return; } if (node.type === 'class_declaration') { const nameNode = node.childForFieldName('name'); if (nameNode !== null) { functions.add(`class:${nodeText(source, nameNode)}`); } return; } if (node.type === 'variable_declarator' && node.parent?.type === 'lexical_declaration') { const valueNode = node.childForFieldName('value'); if (valueNode?.type !== 'arrow_function') { return; } const nameNode = node.childForFieldName('name'); if (nameNode !== null) { functions.add(nodeText(source, nameNode)); } } } function extractPythonImportModules(text: string): string[] { const stripped = text.replace(/^\s*import\s+/, '').trim(); if (stripped.length === 0) { return []; } return stripped .split(',') .map((part) => part.trim()) .map((part) => part.split(/\s+as\s+/i)[0]?.trim() ?? '') .filter((part) => part.length > 0); } function extractPythonFromModule(text: string, relativeFilePath: string): string | null { const match = /^\s*from\s+([^\s]+)\s+import\s+(.+)$/s.exec(text.trim()); if (match === null) { return null; } const sourcePart = match[1].trim(); const importedPart = match[2].trim(); if (!sourcePart.startsWith('.')) { return sourcePart; } const importedNames = importedPart .split(',') .map((item) => item.trim()) .map((item) => item.split(/\s+as\s+/i)[0]?.trim() ?? '') .filter((item) => item.length > 0 && item !== '*'); const base = resolveRelativePythonImport(sourcePart, relativeFilePath); if (base.length === 0) { return importedNames[0] ?? null; } if (sourcePart === '.' && importedNames.length > 0) { return `${base}.${importedNames[0]}`; } return base; } function resolveRelativePythonImport(sourcePart: string, relativeFilePath: string): string { const dotCount = sourcePart.match(/^\.+/)?.[0].length ?? 0; const trailing = sourcePart.slice(dotCount); const parentParts = toPosix(relativeFilePath).split('/').slice(0, -1); const goUp = Math.max(dotCount - 1, 0); let targetParts = parentParts; for (let index = 0; index < goUp; index += 1) { targetParts = targetParts.slice(0, -1); } if (trailing.length > 0) { targetParts = [...targetParts, ...trailing.split('.')]; } return targetParts.filter((part) => part.length > 0).join('.'); } function resolveImportEdges( projectPath: string, parsedByFile: Map, fileSet: Set, ): KnowledgeGraphEdge[] { const edges: KnowledgeGraphEdge[] = []; const edgeSet = new Set(); for (const [fromFile, parsed] of parsedByFile) { for (const rawImportPath of parsed.imports) { if (isBuiltinModule(rawImportPath)) { continue; } const resolved = resolveImportPath(projectPath, fromFile, rawImportPath, fileSet); if (resolved === null) { continue; } if (!fileSet.has(resolved)) { continue; } const edgeKey = `${fromFile}::${resolved}`; if (edgeSet.has(edgeKey)) { continue; } edgeSet.add(edgeKey); edges.push({ from: fromFile, to: resolved, type: EDGE_TYPE_IMPORTS, }); } } return edges; } function resolveImportPath( projectPath: string, fromFile: string, importPath: string, fileSet: Set, ): string | null { const normalizedImport = normalizeImportPath(importPath); if (normalizedImport.length === 0) { return null; } if (normalizedImport.startsWith('./') || normalizedImport.startsWith('../')) { return resolveRelativeImport(projectPath, fromFile, normalizedImport, fileSet); } const absoluteResolved = resolveAbsoluteImport(projectPath, normalizedImport, fileSet); if (absoluteResolved !== null) { return absoluteResolved; } if (fileSet.has(normalizedImport)) { return normalizedImport; } return null; } function resolveRelativeImport( projectPath: string, fromFile: string, importPath: string, fileSet: Set, ): string | null { const fromDir = resolve(projectPath, fromFile, '..'); const target = resolve(fromDir, importPath); const relativeTarget = toPosix(relative(projectPath, target)); const candidates = [ relativeTarget, `${relativeTarget}.ts`, `${relativeTarget}.tsx`, `${relativeTarget}.js`, `${relativeTarget}.py`, `${relativeTarget}/index.ts`, `${relativeTarget}/index.js`, `${relativeTarget}/__init__.py`, ]; for (const candidate of candidates) { if (fileSet.has(candidate)) { return candidate; } } return null; } function resolveAbsoluteImport( projectPath: string, importPath: string, fileSet: Set, ): string | null { const normalized = importPath.replace(/\./g, '/'); const candidates = [ normalized, `${normalized}.py`, `${normalized}.ts`, `${normalized}.tsx`, `${normalized}.js`, `${normalized}.jsx`, `${normalized}/index.py`, `${normalized}/index.ts`, `${normalized}/index.js`, `${normalized}/__init__.py`, ]; for (const candidate of candidates) { if (fileSet.has(candidate)) { return candidate; } const asFsPath = toPosix(relative(projectPath, resolve(projectPath, candidate))); if (fileSet.has(asFsPath)) { return asFsPath; } } return null; } function isBuiltinModule(importPath: string): boolean { const normalized = normalizeImportPath(importPath); if (normalized.startsWith('node:')) { return true; } const rootSegment = normalized.split('/')[0] ?? normalized; return BUILTIN_MODULES.has(rootSegment); } function normalizeImportPath(value: string): string { const strippedQuery = value.split('?')[0] ?? ''; const strippedHash = strippedQuery.split('#')[0] ?? ''; return strippedHash.trim(); } function traverseNode(node: TreeSitterNode, visit: (node: TreeSitterNode) => void): void { visit(node); for (const child of node.namedChildren) { traverseNode(child, visit); } } function nodeText(source: string, node: TreeSitterNode): string { return source.slice(node.startIndex, node.endIndex); } function inferLayer(filePath: string): string { const normalized = toPosix(filePath).toLowerCase(); if (normalized.includes('/api/') || normalized.includes('/routes/')) { return 'api'; } if (normalized.includes('/service/') || normalized.includes('/services/')) { return 'service'; } if (normalized.includes('/db/') || normalized.includes('/models/') || normalized.includes('/database/')) { return 'data'; } if (normalized.includes('/components/') || normalized.includes('/pages/') || normalized.includes('/ui/')) { return 'ui'; } if (normalized.includes('/utils/') || normalized.includes('/helpers/') || normalized.includes('/lib/')) { return 'utility'; } return 'unknown'; } function isExplicitlyExcludedFile(fileName: string): boolean { const lower = fileName.toLowerCase(); if (lower.endsWith('.d.ts')) { return true; } if (lower.endsWith('.min.js')) { return true; } if (/\.test\./i.test(fileName) || /\.spec\./i.test(fileName)) { return true; } return false; } function isExplicitlyExcludedDir(directoryPath: string, projectPath: string): boolean { const relativePath = toPosix(relative(projectPath, directoryPath)).toLowerCase(); const parts = relativePath.split('/'); return parts.includes('obj') || parts.includes('bin'); } function toPosix(pathValue: string): string { return normalize(pathValue).split(sep).join('/'); } function logWarning(onProgress: ((message: string) => void) | undefined, message: string): void { onProgress?.(`⚠️ ${message}`); } function errorToMessage(error: unknown): string { if (error instanceof Error) { return error.message; } return String(error); }