"""Generic AST symbol extractor using tree-sitter.""" import bisect import re from typing import Any, Optional from tree_sitter_language_pack import get_parser from .symbols import Symbol, make_symbol_id, compute_content_hash from .languages import LanguageSpec, LANGUAGE_REGISTRY from .complexity import compute_complexity # Node types that represent function/call expressions per language. # These are used to extract call_references from the AST. _CALL_NODE_TYPES: dict[str, set[str]] = { "python": {"call"}, "javascript": {"call_expression", "new_expression"}, "typescript": {"call_expression", "new_expression"}, "tsx": {"call_expression", "new_expression"}, "go": {"call_expression"}, "rust": {"call_expression"}, "java": {"method_invocation", "object_creation_expression"}, "php": {"function_call_expression", "method_call_expression", "scoped_call_expression"}, "ruby": {"call", "method_call"}, "csharp": {"invocation_expression"}, "kotlin": {"call_expression"}, "dart": {"function_expression_invocation"}, "swift": {"call_expression"}, } def _extract_call_name(node, source_bytes: bytes) -> Optional[str]: """Extract the function/method name from a call node. Handles: - Simple identifier: foo() -> "foo" - Member expression: obj.method() -> "method" - Constructor: new Foo() -> "Foo" - Return None for complex computed calls. """ node_type = node.type if node_type == "identifier": # Simple call: foo() return node.text.decode("utf-8", errors="replace") if node_type in ("call_expression", "function_call_expression", "method_invocation", "invocation_expression", "call", "method_call", "function_expression_invocation", "new_expression", "object_creation_expression"): # For call_expression, the function being called is the first child # For Python call: foo() -> the "foo" is the first child (an identifier) # For JS call_expression: the function is first child (could be identifier or member expression) first_child = None for child in node.children: if child.type not in ("(", ")", "[", "]", "new"): first_child = child break if first_child is None: return None ft = first_child.type if ft in ("identifier", "type_identifier"): return first_child.text.decode("utf-8", errors="replace") elif ft in ("member_expression", "attribute_expression", "attribute", "method_declaration"): # For JS/TS: member_expression contains property_identifier for the method name # For Python: attribute node contains two identifiers (object and method) # First check for property_identifier (JS/TS way) for child in first_child.children: if child.type == "property_identifier": return child.text.decode("utf-8", errors="replace") # Fallback: for Python attribute, get the last identifier (method name) identifiers = [c for c in first_child.children if c.type == "identifier"] if identifiers: return identifiers[-1].text.decode("utf-8", errors="replace") elif ft == "call_expression": # Nested call: foo()(bar) - extract foo's name return _extract_call_name(first_child, source_bytes) else: # Could be a parenthesized expression or other complex case # Try to find an identifier within for child in first_child.children: if child.type == "identifier": return child.text.decode("utf-8", errors="replace") return None def _collect_calls( node, call_types: set[str], source_bytes: bytes, results: list[tuple[int, str]], ) -> None: """Iteratively walk AST collecting call nodes using explicit stack. Uses an explicit stack to avoid Python's recursion limit on deeply nested or generated code. Args: node: Current AST node (used as the initial stack entry) call_types: Set of node type names that represent calls source_bytes: Source bytes for decoding text results: Out list of (byte_offset, called_name) tuples """ stack = [node] while stack: current = stack.pop() if current.type in call_types: name = _extract_call_name(current, source_bytes) if name: results.append((current.start_byte, name)) # Extend with all children at once (push in reverse for pre-order) stack.extend(reversed(current.children)) def _find_enclosing_symbol( sorted_syms: list[tuple[int, int, int, Symbol]], byte_offset: int, ) -> Optional[Symbol]: """Find the symbol that contains the given byte offset. Does a linear scan backwards from the binary-search candidate to find the innermost enclosing symbol. Args: sorted_syms: List of (byte_offset, byte_end, line, symbol) sorted by byte_offset byte_offset: Byte offset to find enclosing symbol for Returns: The Symbol that contains this byte offset, or None """ if not sorted_syms: return None # Binary search for the last symbol whose start <= byte_offset starts = [s[0] for s in sorted_syms] idx = bisect.bisect_right(starts, byte_offset) - 1 # Scan backwards to find the innermost enclosing symbol while idx >= 0: start, end, line, sym = sorted_syms[idx] if start <= byte_offset <= end: return sym idx -= 1 return None def _attribute_calls_to_symbols( symbols: list[Symbol], calls: list[tuple[int, str]], ) -> None: """Attribute pre-collected call sites to their enclosing symbols. This is the cheap second step after call sites have been collected (either during ``_walk_tree`` or via ``_collect_calls``). Only builds the sorted symbol list and does bisect lookups — no AST walk. """ if not calls: return callable_syms = [ (s.byte_offset, s.byte_offset + s.byte_length, s.line, s) for s in symbols if s.kind in ("function", "method") and s.byte_offset >= 0 ] callable_syms.sort(key=lambda x: x[0]) if not callable_syms: return for call_offset, called_name in calls: enclosing = _find_enclosing_symbol(callable_syms, call_offset) if enclosing and enclosing.name != called_name: if called_name not in enclosing.call_references: enclosing.call_references.append(called_name) def _extract_call_references( root_node, symbols: list[Symbol], source_bytes: bytes, language: str, ) -> None: """Extract call references via a standalone AST walk (for custom parsers). Used by custom parsers (C++, Elixir, etc.) that don't go through ``_parse_with_spec`` / ``_walk_tree``. The generic path uses ``_walk_tree(call_types=..., calls=...)`` instead to avoid a second walk. """ call_types = _CALL_NODE_TYPES.get(language) if not call_types: return calls: list[tuple[int, str]] = [] _collect_calls(root_node, call_types, source_bytes, calls) _attribute_calls_to_symbols(symbols, calls) def parse_file(content: str, filename: str, language: str, source_bytes: Optional[bytes] = None, repo: Optional[str] = None) -> list[Symbol]: """Parse source code and extract symbols using tree-sitter. Args: content: Raw source code filename: File path (for ID generation) language: Language name (must be in LANGUAGE_REGISTRY) source_bytes: Optional pre-encoded UTF-8 bytes. If provided, avoids a redundant encode() call when the caller has already encoded content. repo: Optional folder path used to consult per-project .jcodemunch.jsonc when checking whether the language is enabled. Returns: List of Symbol objects """ if language not in LANGUAGE_REGISTRY: return [] # Skip parsing if the language is not in the configured languages list. # When languages config is None (default), all languages are enabled. # Pass repo so that per-project .jcodemunch.jsonc overrides the global config. try: from ..config import is_language_enabled as _is_lang_enabled if not _is_lang_enabled(language, repo=repo): return [] except ImportError: pass # config module not available (e.g. standalone use) if source_bytes is None: source_bytes = content.encode("utf-8") # Track the tree for call reference extraction (custom parsers may return it) root_node: Any = None if language == "cpp": symbols, root_node = _parse_cpp_symbols(source_bytes, filename) elif language == "elixir": symbols = _parse_elixir_symbols(source_bytes, filename) elif language == "blade": symbols = _parse_blade_symbols(source_bytes, filename) elif language == "razor": symbols = _parse_razor_symbols(source_bytes, filename) elif language == "nix": symbols = _parse_nix_symbols(source_bytes, filename) elif language == "vue": symbols = _parse_vue_symbols(source_bytes, filename) elif language == "ejs": symbols = _parse_ejs_symbols(source_bytes, filename) elif language == "verse": symbols = _parse_verse_symbols(source_bytes, filename) elif language == "lua": symbols = _parse_lua_symbols(source_bytes, filename) elif language == "luau": symbols = _parse_luau_symbols(source_bytes, filename) elif language == "erlang": symbols = _parse_erlang_symbols(source_bytes, filename) elif language == "fortran": symbols = _parse_fortran_symbols(source_bytes, filename) elif language == "sql": symbols = _parse_sql_symbols(source_bytes, filename) elif language == "objc": symbols = _parse_objc_symbols(source_bytes, filename) elif language == "proto": symbols = _parse_proto_symbols(source_bytes, filename) elif language == "hcl": symbols = _parse_hcl_symbols(source_bytes, filename) elif language == "graphql": symbols = _parse_graphql_symbols(source_bytes, filename) elif language == "julia": symbols = _parse_julia_symbols(source_bytes, filename) elif language == "groovy": symbols = _parse_groovy_symbols(source_bytes, filename) elif language == "autohotkey": symbols = _parse_autohotkey_symbols(source_bytes, filename) elif language == "asm": symbols = _parse_asm_symbols(source_bytes, filename) elif language == "xml": symbols = _parse_xml_symbols(source_bytes, filename) elif language == "yaml": symbols = _parse_yaml_symbols(source_bytes, filename) elif language == "ansible": symbols = _parse_ansible_symbols(source_bytes, filename) elif language == "openapi": symbols = _parse_openapi_symbols(source_bytes, filename) elif language == "al": symbols = _parse_al_symbols(source_bytes, filename) elif language == "css": symbols = _parse_css_symbols(source_bytes, filename) elif language == "scss": symbols = _parse_scss_symbols(source_bytes, filename) elif language in ("sass", "less", "styl"): symbols = [] # No tree-sitter grammar; files indexed for text search only elif language == "json": symbols = _parse_json_symbols(source_bytes, filename) else: spec = LANGUAGE_REGISTRY[language] symbols = _parse_with_spec(source_bytes, filename, language, spec) # _parse_with_spec calls _extract_call_references internally root_node = None # already handled inside _parse_with_spec # Extract call references for custom parsers that created a tree if root_node is not None: _extract_call_references(root_node, symbols, source_bytes, language) # Disambiguate overloaded symbols + compute complexity in a single pass symbols = _disambiguate_and_compute_complexity(symbols, source_bytes) return symbols def _parse_with_spec( source_bytes: bytes, filename: str, language: str, spec: LanguageSpec, ) -> list[Symbol]: """Parse source bytes using one language spec.""" try: parser = get_parser(spec.ts_language) tree = parser.parse(source_bytes) except Exception: return [] symbols: list[Symbol] = [] # Collect call sites during the same walk as symbol extraction (single pass). ct = _CALL_NODE_TYPES.get(language) calls: list[tuple[int, str]] = [] if ct else [] _walk_tree(tree.root_node, spec, source_bytes, filename, language, symbols, None, call_types=ct, calls=calls if ct else None) # Attribute collected call sites to enclosing symbols (cheap — no AST walk) if calls: _attribute_calls_to_symbols(symbols, calls) return symbols def _parse_cpp_symbols(source_bytes: bytes, filename: str) -> tuple[list[Symbol], Any]: """Parse C++ and auto-fallback to C for `.h` files with no C++ symbols. Returns (symbols, root_node) tuple so parse_file can call _extract_call_references. """ cpp_spec = LANGUAGE_REGISTRY["cpp"] cpp_symbols: list[Symbol] = [] cpp_error_nodes = 0 cpp_tree: Any = None try: parser = get_parser(cpp_spec.ts_language) tree = parser.parse(source_bytes) cpp_tree = tree cpp_error_nodes = _count_error_nodes(tree.root_node) _walk_tree(tree.root_node, cpp_spec, source_bytes, filename, "cpp", cpp_symbols, None) except Exception: cpp_error_nodes = 10**9 # Non-headers are always C++. if not filename.lower().endswith(".h"): return cpp_symbols, cpp_tree # Header auto-detection: parse both C++ and C, prefer better parse quality. c_spec = LANGUAGE_REGISTRY.get("c") if not c_spec: return cpp_symbols, cpp_tree c_symbols: list[Symbol] = [] c_error_nodes = 10**9 c_tree: Any = None try: c_parser = get_parser(c_spec.ts_language) c_tree_obj = c_parser.parse(source_bytes) c_tree = c_tree_obj c_error_nodes = _count_error_nodes(c_tree_obj.root_node) _walk_tree(c_tree_obj.root_node, c_spec, source_bytes, filename, "c", c_symbols, None) except Exception: c_error_nodes = 10**9 # If only one parser yields symbols, use that parser's symbols. if cpp_symbols and not c_symbols: return cpp_symbols, cpp_tree if c_symbols and not cpp_symbols: return c_symbols, c_tree if not cpp_symbols and not c_symbols: return cpp_symbols, cpp_tree # Both yielded symbols: choose fewer parse errors first, then richer symbol output. if c_error_nodes < cpp_error_nodes: return c_symbols, c_tree if cpp_error_nodes < c_error_nodes: return cpp_symbols, cpp_tree # Same error quality: use lexical signal to break ties for `.h`. if _looks_like_cpp_header(source_bytes): if len(cpp_symbols) >= len(c_symbols): return cpp_symbols, cpp_tree else: return c_symbols, c_tree if len(c_symbols) > len(cpp_symbols): return c_symbols, c_tree return cpp_symbols, cpp_tree def _walk_tree( node, spec: LanguageSpec, source_bytes: bytes, filename: str, language: str, symbols: list, parent_symbol: Optional[Symbol] = None, scope_parts: Optional[list[str]] = None, class_scope_depth: int = 0, call_types: Optional[set[str]] = None, calls: Optional[list] = None, ): """Recursively walk the AST and extract symbols. When *call_types* and *calls* are provided, also collects call sites (byte_offset, called_name) in a single pass — no second AST walk needed. """ # Dart: function_signature inside method_signature is handled by method_signature if node.type == "function_signature" and node.parent and node.parent.type == "method_signature": return is_cpp = language == "cpp" local_scope_parts = scope_parts or [] next_parent = parent_symbol next_class_scope_depth = class_scope_depth if is_cpp and node.type == "namespace_definition": ns_name = _extract_cpp_namespace_name(node, source_bytes) if ns_name: local_scope_parts = [*local_scope_parts, ns_name] # Collect call sites during the same walk (when enabled) if call_types is not None and calls is not None and node.type in call_types: name = _extract_call_name(node, source_bytes) if name: calls.append((node.start_byte, name)) # Check if this node is a symbol if node.type in spec.symbol_node_types: # C++ declarations include non-function declarations. Filter those out. if not (is_cpp and node.type in {"declaration", "field_declaration"} and not _is_cpp_function_declaration(node)): symbol = _extract_symbol( node, spec, source_bytes, filename, language, parent_symbol, local_scope_parts, class_scope_depth, ) if symbol: symbols.append(symbol) if is_cpp: if _is_cpp_type_container(node): next_parent = symbol next_class_scope_depth = class_scope_depth + 1 else: next_parent = symbol # Check for arrow/function-expression variable assignments in JS/TS if node.type == "variable_declarator" and language in ("javascript", "typescript", "tsx"): var_func = _extract_variable_function( node, spec, source_bytes, filename, language, parent_symbol ) if var_func: symbols.append(var_func) # Check for constant patterns (top-level assignments with UPPER_CASE names) if node.type in spec.constant_patterns and parent_symbol is None: const_symbol = _extract_constant(node, spec, source_bytes, filename, language) if const_symbol: symbols.append(const_symbol) # Recurse into children for child in node.children: _walk_tree( child, spec, source_bytes, filename, language, symbols, next_parent, local_scope_parts, next_class_scope_depth, call_types, calls, ) def _extract_symbol( node, spec: LanguageSpec, source_bytes: bytes, filename: str, language: str, parent_symbol: Optional[Symbol] = None, scope_parts: Optional[list[str]] = None, class_scope_depth: int = 0, ) -> Optional[Symbol]: """Extract a Symbol from an AST node.""" kind = spec.symbol_node_types[node.type] # Skip nodes with errors if node.has_error: return None # Extract name name = _extract_name(node, spec, source_bytes) if not name: return None # Build qualified name if language == "cpp": if parent_symbol: qualified_name = f"{parent_symbol.qualified_name}.{name}" elif scope_parts: qualified_name = ".".join([*scope_parts, name]) else: qualified_name = name if kind == "function" and class_scope_depth > 0: kind = "method" else: if parent_symbol: qualified_name = f"{parent_symbol.name}.{name}" kind = "method" if kind == "function" else kind else: qualified_name = name signature_node = node if language == "cpp": wrapper = _nearest_cpp_template_wrapper(node) if wrapper: signature_node = wrapper # Build signature signature = _build_signature(signature_node, spec, source_bytes) # Extract docstring docstring = _extract_docstring(signature_node, spec, source_bytes) # Extract decorators decorators = _extract_decorators(node, spec, source_bytes) start_node = signature_node # Dart: function_signature/method_signature have their body as a next sibling end_byte = node.end_byte end_line_num = node.end_point[0] + 1 if node.type in ("function_signature", "method_signature"): next_sib = node.next_named_sibling if next_sib and next_sib.type == "function_body": end_byte = next_sib.end_byte end_line_num = next_sib.end_point[0] + 1 # Compute content hash symbol_bytes = source_bytes[start_node.start_byte:end_byte] c_hash = compute_content_hash(symbol_bytes) # Create symbol symbol = Symbol( id=make_symbol_id(filename, qualified_name, kind), file=filename, name=name, qualified_name=qualified_name, kind=kind, language=language, signature=signature, docstring=docstring, decorators=decorators, parent=parent_symbol.id if parent_symbol else None, line=start_node.start_point[0] + 1, end_line=end_line_num, byte_offset=start_node.start_byte, byte_length=end_byte - start_node.start_byte, content_hash=c_hash, ) return symbol def _extract_name(node, spec: LanguageSpec, source_bytes: bytes) -> Optional[str]: """Extract the name from an AST node.""" # Handle type_declaration in Go - name is in type_spec child if node.type == "type_declaration": for child in node.children: if child.type == "type_spec": name_node = child.child_by_field_name("name") if name_node: return source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") return None # Dart: mixin_declaration has identifier as direct child (no field name) if node.type == "mixin_declaration": for child in node.children: if child.type == "identifier": return source_bytes[child.start_byte:child.end_byte].decode("utf-8") return None # Dart: method_signature wraps function_signature or getter_signature if node.type == "method_signature": for child in node.children: if child.type in ("function_signature", "getter_signature"): name_node = child.child_by_field_name("name") if name_node: return source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") return None # Dart: type_alias name is the first type_identifier child if node.type == "type_alias" and spec.ts_language == "dart": for child in node.children: if child.type == "type_identifier": return source_bytes[child.start_byte:child.end_byte].decode("utf-8") return None # Kotlin: no named fields; walk children by type to find name if spec.ts_language == "kotlin": if node.type in ("class_declaration", "object_declaration", "type_alias"): for child in node.children: if child.type == "type_identifier": return source_bytes[child.start_byte:child.end_byte].decode("utf-8") return None if node.type == "function_declaration": for child in node.children: if child.type == "simple_identifier": return source_bytes[child.start_byte:child.end_byte].decode("utf-8") return None # Gleam: type_definition and type_alias names live inside a type_name child if spec.ts_language == "gleam" and node.type in ("type_definition", "type_alias"): for child in node.children: if child.type == "type_name": name_node = child.child_by_field_name("name") if name_node: return source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") return None # C#: field_declaration and event_field_declaration wrappers if spec.ts_language == "csharp" and node.type in ("field_declaration", "event_field_declaration"): for child in node.children: if child.type == "variable_declaration": # Find the first variable_declarator child for vdecl in child.children: if vdecl.type == "variable_declarator": name_node = vdecl.child_by_field_name("name") if name_node: return source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") return None if node.type not in spec.name_fields: return None field_name = spec.name_fields[node.type] name_node = node.child_by_field_name(field_name) if name_node: if spec.ts_language == "cpp": return _extract_cpp_name(name_node, source_bytes) # C function_definition: declarator is a function_declarator, # which wraps the actual identifier. Unwrap recursively. while name_node.type in ("function_declarator", "pointer_declarator", "reference_declarator"): inner = name_node.child_by_field_name("declarator") if inner: name_node = inner else: break return source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") return None def _extract_cpp_name(name_node, source_bytes: bytes) -> Optional[str]: """Extract C++ symbol names from nested declarators.""" current = name_node wrapper_types = { "function_declarator", "pointer_declarator", "reference_declarator", "array_declarator", "parenthesized_declarator", "attributed_declarator", "init_declarator", } while current.type in wrapper_types: inner = current.child_by_field_name("declarator") if not inner: break current = inner # Prefer typed name children where available. if current.type in {"qualified_identifier", "scoped_identifier"}: name_node = current.child_by_field_name("name") if name_node: text = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8").strip() if text: return text subtree_name = _find_cpp_name_in_subtree(current, source_bytes) if subtree_name: return subtree_name text = source_bytes[current.start_byte:current.end_byte].decode("utf-8").strip() return text or None def _find_cpp_name_in_subtree(node, source_bytes: bytes) -> Optional[str]: """Best-effort extraction of a callable/type name from a declarator subtree.""" direct_types = {"identifier", "field_identifier", "operator_name", "destructor_name", "type_identifier"} if node.type in direct_types: text = source_bytes[node.start_byte:node.end_byte].decode("utf-8").strip() return text or None if node.type in {"qualified_identifier", "scoped_identifier"}: name_node = node.child_by_field_name("name") if name_node: return _find_cpp_name_in_subtree(name_node, source_bytes) for child in node.children: if not child.is_named: continue found = _find_cpp_name_in_subtree(child, source_bytes) if found: return found return None def _build_signature(node, spec: LanguageSpec, source_bytes: bytes) -> str: """Build a clean signature from AST node.""" if node.type == "template_declaration": inner = node.child_by_field_name("declaration") if not inner: for child in reversed(node.children): if child.is_named: inner = child break if inner: body = inner.child_by_field_name("body") end_byte = body.start_byte if body else inner.end_byte else: end_byte = node.end_byte elif spec.ts_language == "csharp" and node.type == "property_declaration": # C# properties use 'accessors' field instead of 'body' body = node.child_by_field_name("accessors") end_byte = body.start_byte if body else node.end_byte elif spec.ts_language == "kotlin": # Kotlin uses no named fields; find body child by type body = None for child in node.children: if child.type in ("function_body", "class_body", "enum_class_body"): body = child break end_byte = body.start_byte if body else node.end_byte else: # Find the body child to determine where signature ends body = node.child_by_field_name("body") if body: # Signature is from start of node to start of body end_byte = body.start_byte else: end_byte = node.end_byte sig_bytes = source_bytes[node.start_byte:end_byte] sig_text = sig_bytes.decode("utf-8").strip() # Clean up: remove trailing '{', ':', etc. sig_text = sig_text.rstrip("{: \n\t") return sig_text def _nearest_cpp_template_wrapper(node): """Return closest enclosing template_declaration (if any).""" current = node wrapper = None while current.parent and current.parent.type == "template_declaration": wrapper = current.parent current = current.parent return wrapper def _is_cpp_type_container(node) -> bool: """C++ node types that can contain methods.""" return node.type in {"class_specifier", "struct_specifier", "union_specifier"} def _is_cpp_function_declaration(node) -> bool: """True if a C++ declaration node is function-like.""" if node.type not in {"declaration", "field_declaration"}: return True declarator = node.child_by_field_name("declarator") if not declarator: return False return _has_function_declarator(declarator) def _has_function_declarator(node) -> bool: """Check subtree for function declarator nodes.""" if node.type in {"function_declarator", "abstract_function_declarator"}: return True for child in node.children: if child.is_named and _has_function_declarator(child): return True return False def _extract_cpp_namespace_name(node, source_bytes: bytes) -> Optional[str]: """Extract namespace name from a namespace_definition node.""" name_node = node.child_by_field_name("name") if not name_node: for child in node.children: if child.type in {"namespace_identifier", "identifier"}: name_node = child break if not name_node: return None name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8").strip() return name or None def _looks_like_cpp_header(source_bytes: bytes) -> bool: """Heuristic: detect obvious C++ constructs in `.h` content.""" text = source_bytes.decode("utf-8", errors="ignore") cpp_markers = ( "namespace ", "class ", "template<", "template <", "constexpr", "noexcept", "[[", "std::", "using ", "::", "public:", "private:", "protected:", "operator", "typename", ) return any(marker in text for marker in cpp_markers) def _count_error_nodes(node) -> int: """Count parser ERROR nodes in a syntax tree subtree.""" count = 1 if node.type == "ERROR" else 0 for child in node.children: count += _count_error_nodes(child) return count def _extract_docstring(node, spec: LanguageSpec, source_bytes: bytes) -> str: """Extract docstring using language-specific strategy.""" if spec.docstring_strategy == "next_sibling_string": return _extract_python_docstring(node, source_bytes) elif spec.docstring_strategy == "preceding_comment": return _extract_preceding_comments(node, source_bytes) return "" def _extract_python_docstring(node, source_bytes: bytes) -> str: """Extract Python docstring from first statement in body.""" body = node.child_by_field_name("body") if not body or body.child_count == 0: return "" # Find first expression_statement in body (function docstrings) for child in body.children: if child.type == "expression_statement": # Check if it's a string expr = child.child_by_field_name("expression") if expr and expr.type == "string": doc = source_bytes[expr.start_byte:expr.end_byte].decode("utf-8") return _strip_quotes(doc) # Handle tree-sitter-python 0.21+ string format if child.child_count > 0: first = child.children[0] if first.type in ("string", "concatenated_string"): doc = source_bytes[first.start_byte:first.end_byte].decode("utf-8") return _strip_quotes(doc) # Class docstrings are directly string nodes in the block elif child.type == "string": doc = source_bytes[child.start_byte:child.end_byte].decode("utf-8") return _strip_quotes(doc) return "" def _strip_quotes(text: str) -> str: """Strip quotes from a docstring.""" text = text.strip() if text.startswith('"""') and text.endswith('"""'): return text[3:-3].strip() if text.startswith("'''") and text.endswith("'''"): return text[3:-3].strip() if text.startswith('"') and text.endswith('"'): return text[1:-1].strip() if text.startswith("'") and text.endswith("'"): return text[1:-1].strip() return text def _extract_preceding_comments(node, source_bytes: bytes) -> str: """Extract comments that immediately precede a node.""" comments = [] # Walk backwards through siblings, skipping past annotations/decorators prev = node.prev_named_sibling while prev and prev.type in ("annotation", "marker_annotation"): prev = prev.prev_named_sibling while prev and prev.type in ("comment", "line_comment", "block_comment", "documentation_comment", "pod"): comment_text = source_bytes[prev.start_byte:prev.end_byte].decode("utf-8") comments.insert(0, comment_text) prev = prev.prev_named_sibling if not comments: return "" docstring = "\n".join(comments) return _clean_comment_markers(docstring) def _clean_comment_markers(text: str) -> str: """Clean comment markers from docstring.""" # POD block: strip directive lines (=pod, =head1, =cut, etc.), keep content if text.lstrip().startswith("="): content_lines = [] for line in text.split("\n"): stripped = line.strip() if stripped.startswith("="): continue content_lines.append(stripped) return "\n".join(content_lines).strip() lines = text.split("\n") cleaned = [] for line in lines: line = line.strip() # Remove leading comment markers (order matters: longer prefixes first) if line.startswith("/**"): line = line[3:] elif line.startswith("//!"): line = line[3:] elif line.startswith("///"): line = line[3:] elif line.startswith("//"): line = line[2:] elif line.startswith("/*"): line = line[2:] elif line.startswith("*"): line = line[1:] elif line.startswith("#"): line = line[1:] # Remove trailing */ if line.endswith("*/"): line = line[:-2] cleaned.append(line.strip()) return "\n".join(cleaned).strip() def _extract_decorators(node, spec: LanguageSpec, source_bytes: bytes) -> list[str]: """Extract decorators/attributes from a node.""" if not spec.decorator_node_type: return [] decorators = [] if spec.decorator_from_children: # C#: attribute_list nodes are direct children of the declaration for child in node.children: if child.type == spec.decorator_node_type: decorator_text = source_bytes[child.start_byte:child.end_byte].decode("utf-8") decorators.append(decorator_text.strip()) else: # Other languages: decorators are preceding siblings prev = node.prev_named_sibling while prev and prev.type == spec.decorator_node_type: decorator_text = source_bytes[prev.start_byte:prev.end_byte].decode("utf-8") decorators.insert(0, decorator_text.strip()) prev = prev.prev_named_sibling return decorators _VARIABLE_FUNCTION_TYPES = frozenset({ "arrow_function", "function_expression", "generator_function", }) def _extract_variable_function( node, spec: LanguageSpec, source_bytes: bytes, filename: str, language: str, parent_symbol: Optional[Symbol] = None, ) -> Optional[Symbol]: """Extract a function from `const name = () => {}` or `const name = function() {}`.""" # node is a variable_declarator name_node = node.child_by_field_name("name") if not name_node or name_node.type != "identifier": return None # destructuring or other non-simple binding value_node = node.child_by_field_name("value") if not value_node or value_node.type not in _VARIABLE_FUNCTION_TYPES: return None # not a function assignment name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") kind = "function" if parent_symbol: qualified_name = f"{parent_symbol.name}.{name}" kind = "method" else: qualified_name = name # Signature: use the full declaration statement (lexical_declaration parent) # to capture export/const keywords sig_node = node.parent if node.parent and node.parent.type in ( "lexical_declaration", "export_statement", "variable_declaration", ) else node # Walk up through export_statement wrapper if present if sig_node.parent and sig_node.parent.type == "export_statement": sig_node = sig_node.parent signature = _build_signature(sig_node, spec, source_bytes) # Docstring: look for preceding comment on the declaration statement doc_node = sig_node docstring = _extract_docstring(doc_node, spec, source_bytes) # Content hash covers the full declaration start_byte = sig_node.start_byte end_byte = sig_node.end_byte symbol_bytes = source_bytes[start_byte:end_byte] c_hash = compute_content_hash(symbol_bytes) return Symbol( id=make_symbol_id(filename, qualified_name, kind), file=filename, name=name, qualified_name=qualified_name, kind=kind, language=language, signature=signature, docstring=docstring, parent=parent_symbol.id if parent_symbol else None, line=sig_node.start_point[0] + 1, end_line=sig_node.end_point[0] + 1, byte_offset=start_byte, byte_length=end_byte - start_byte, content_hash=c_hash, ) def _extract_constant( node, spec: LanguageSpec, source_bytes: bytes, filename: str, language: str ) -> Optional[Symbol]: """Extract a constant (UPPER_CASE top-level assignment).""" # Only extract constants at module level for Python if node.type == "assignment": left = node.child_by_field_name("left") if left and left.type == "identifier": name = source_bytes[left.start_byte:left.end_byte].decode("utf-8") # Check if UPPER_CASE (constant convention) if name.isupper() or (len(name) > 1 and name[0].isupper() and "_" in name): # Get the full assignment text as signature sig = source_bytes[node.start_byte:node.end_byte].decode("utf-8").strip() const_bytes = source_bytes[node.start_byte:node.end_byte] c_hash = compute_content_hash(const_bytes) return Symbol( id=make_symbol_id(filename, name, "constant"), file=filename, name=name, qualified_name=name, kind="constant", language=language, signature=sig[:100], # Truncate long assignments line=node.start_point[0] + 1, end_line=node.end_point[0] + 1, byte_offset=node.start_byte, byte_length=node.end_byte - node.start_byte, content_hash=c_hash, ) # C preprocessor #define macros if node.type == "preproc_def": name_node = node.child_by_field_name("name") if name_node: name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") if name.isupper() or (len(name) > 1 and name[0].isupper() and "_" in name): sig = source_bytes[node.start_byte:node.end_byte].decode("utf-8").strip() const_bytes = source_bytes[node.start_byte:node.end_byte] c_hash = compute_content_hash(const_bytes) return Symbol( id=make_symbol_id(filename, name, "constant"), file=filename, name=name, qualified_name=name, kind="constant", language=language, signature=sig[:100], line=node.start_point[0] + 1, end_line=node.end_point[0] + 1, byte_offset=node.start_byte, byte_length=node.end_byte - node.start_byte, content_hash=c_hash, ) # GDScript: const MAX_SPEED: float = 100.0 (all const declarations are constants) if node.type == "const_statement": name_node = node.child_by_field_name("name") if name_node: name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") sig = source_bytes[node.start_byte:node.end_byte].decode("utf-8").strip() const_bytes = source_bytes[node.start_byte:node.end_byte] c_hash = compute_content_hash(const_bytes) return Symbol( id=make_symbol_id(filename, name, "constant"), file=filename, name=name, qualified_name=name, kind="constant", language=language, signature=sig[:100], line=node.start_point[0] + 1, end_line=node.end_point[0] + 1, byte_offset=node.start_byte, byte_length=node.end_byte - node.start_byte, content_hash=c_hash, ) # Perl: use constant NAME => value if node.type == "use_statement": children = list(node.children) if len(children) >= 3 and children[1].type == "package": pkg_name = source_bytes[children[1].start_byte:children[1].end_byte].decode("utf-8") if pkg_name == "constant": for child in children: if child.type == "list_expression" and child.child_count >= 1: name_node = child.children[0] if name_node.type == "autoquoted_bareword": name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") if name.isupper() or (len(name) > 1 and name[0].isupper()): sig = source_bytes[node.start_byte:node.end_byte].decode("utf-8").strip() const_bytes = source_bytes[node.start_byte:node.end_byte] c_hash = compute_content_hash(const_bytes) return Symbol( id=make_symbol_id(filename, name, "constant"), file=filename, name=name, qualified_name=name, kind="constant", language=language, signature=sig[:100], line=node.start_point[0] + 1, end_line=node.end_point[0] + 1, byte_offset=node.start_byte, byte_length=node.end_byte - node.start_byte, content_hash=c_hash, ) # Swift: let MAX_SPEED = 100 (property_declaration with let binding) if node.type == "property_declaration": # Only extract immutable `let` bindings (not `var`) binding = None for child in node.children: if child.type == "value_binding_pattern": binding = child break if not binding: return None mutability = binding.child_by_field_name("mutability") if not mutability or mutability.text != b"let": return None pattern = node.child_by_field_name("name") if not pattern: return None name_node = pattern.child_by_field_name("bound_identifier") if not name_node: # fallback: first simple_identifier in pattern for child in pattern.children: if child.type == "simple_identifier": name_node = child break if not name_node: return None name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") if not (name.isupper() or (len(name) > 1 and name[0].isupper() and "_" in name)): return None sig = source_bytes[node.start_byte:node.end_byte].decode("utf-8").strip() const_bytes = source_bytes[node.start_byte:node.end_byte] c_hash = compute_content_hash(const_bytes) return Symbol( id=make_symbol_id(filename, name, "constant"), file=filename, name=name, qualified_name=name, kind="constant", language=language, signature=sig[:100], line=node.start_point[0] + 1, end_line=node.end_point[0] + 1, byte_offset=node.start_byte, byte_length=node.end_byte - node.start_byte, content_hash=c_hash, ) # JS/TS/TSX: index `const` declarations as constants. # `export const foo = ...` appears as a lexical_declaration under an export_statement; # plain `const foo = ...` is a lexical_declaration at module scope. # variable_declaration covers `var`/`let` at module scope in some tree-sitter grammars. if node.type in ("lexical_declaration", "variable_declaration"): if language not in ("javascript", "typescript", "tsx"): return None for child in node.children: if child.type != "variable_declarator": continue name_node = child.child_by_field_name("name") if not name_node or name_node.type != "identifier": continue name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8") # Arrow functions and function expressions are handled by _extract_variable_function value_node = child.child_by_field_name("value") if value_node and value_node.type in ( "arrow_function", "function_expression", "generator_function", ): continue sig = source_bytes[node.start_byte:node.end_byte].decode("utf-8").strip() const_bytes = source_bytes[node.start_byte:node.end_byte] c_hash = compute_content_hash(const_bytes) return Symbol( id=make_symbol_id(filename, name, "constant"), file=filename, name=name, qualified_name=name, kind="constant", language=language, signature=sig[:200], line=node.start_point[0] + 1, end_line=node.end_point[0] + 1, byte_offset=node.start_byte, byte_length=node.end_byte - node.start_byte, content_hash=c_hash, ) return None # =========================================================================== # Elixir custom extractor # =========================================================================== def _get_elixir_args(node) -> Optional[object]: """Return the `arguments` named child of an Elixir AST node. The Elixir tree-sitter grammar does not expose `arguments` as a named field (only `target` is a named field on `call` nodes), so we find it by scanning named_children. """ for child in node.named_children: if child.type == "arguments": return child return None # --- Elixir keyword sets --- _ELIXIR_MODULE_KW = frozenset({"defmodule", "defprotocol", "defimpl"}) _ELIXIR_FUNCTION_KW = frozenset({"def", "defp", "defmacro", "defmacrop", "defguard", "defguardp"}) _ELIXIR_TYPE_ATTRS = frozenset({"type", "typep", "opaque"}) _ELIXIR_SKIP_ATTRS = frozenset({"spec", "impl"}) def _node_text(node, source_bytes: bytes) -> str: """Return the decoded text of a tree-sitter node.""" return source_bytes[node.start_byte:node.end_byte].decode("utf-8").strip() def _first_named_child(node): """Return the first named child of a node, or None.""" return next((c for c in node.children if c.is_named), None) def _get_elixir_attr_name(node, source_bytes: bytes) -> Optional[str]: """Extract the attribute name from a unary_operator `@attr` node, or None.""" inner = _first_named_child(node) if inner and inner.type == "call": target = inner.child_by_field_name("target") if target: return _node_text(target, source_bytes) return None def _make_elixir_symbol( node, source_bytes: bytes, filename: str, name: str, qualified_name: str, kind: str, parent_symbol: Optional[Symbol], signature: str, docstring: str = "" ) -> Symbol: """Construct a Symbol for an Elixir node.""" symbol_bytes = source_bytes[node.start_byte:node.end_byte] return Symbol( id=make_symbol_id(filename, qualified_name, kind), file=filename, name=name, qualified_name=qualified_name, kind=kind, language="elixir", signature=signature, docstring=docstring, parent=parent_symbol.id if parent_symbol else None, line=node.start_point[0] + 1, end_line=node.end_point[0] + 1, byte_offset=node.start_byte, byte_length=node.end_byte - node.start_byte, content_hash=compute_content_hash(symbol_bytes), ) def _parse_elixir_symbols(source_bytes: bytes, filename: str) -> list[Symbol]: """Parse Elixir source and return extracted symbols.""" spec = LANGUAGE_REGISTRY["elixir"] try: parser = get_parser(spec.ts_language) tree = parser.parse(source_bytes) except Exception: return [] symbols: list[Symbol] = [] _walk_elixir(tree.root_node, source_bytes, filename, symbols, None) return symbols def _walk_elixir(node, source_bytes: bytes, filename: str, symbols: list, parent_symbol: Optional[Symbol]): """Recursively walk Elixir AST and extract symbols.""" if node.type == "call": target = node.child_by_field_name("target") if target is None: _walk_elixir_children(node, source_bytes, filename, symbols, parent_symbol) return keyword = _node_text(target, source_bytes) if keyword in _ELIXIR_MODULE_KW: sym = _extract_elixir_module(node, keyword, source_bytes, filename, parent_symbol) if sym: symbols.append(sym) # Recurse into do_block with this module as parent do_block = _find_elixir_do_block(node) if do_block: _walk_elixir_children(do_block, source_bytes, filename, symbols, sym) return if keyword in _ELIXIR_FUNCTION_KW: sym = _extract_elixir_function(node, keyword, source_bytes, filename, parent_symbol) if sym: symbols.append(sym) return elif node.type == "unary_operator": inner_call = _first_named_child(node) if inner_call and inner_call.type == "call": inner_target = inner_call.child_by_field_name("target") if inner_target: attr_name = _node_text(inner_target, source_bytes) if attr_name in _ELIXIR_TYPE_ATTRS or attr_name == "callback": sym = _extract_elixir_type_attribute(node, attr_name, inner_call, source_bytes, filename, parent_symbol) if sym: symbols.append(sym) return _walk_elixir_children(node, source_bytes, filename, symbols, parent_symbol) def _walk_elixir_children(node, source_bytes: bytes, filename: str, symbols: list, parent_symbol: Optional[Symbol]): for child in node.children: _walk_elixir(child, source_bytes, filename, symbols, parent_symbol) def _find_elixir_do_block(call_node) -> Optional[object]: """Find the do_block child of a call node.""" for child in call_node.children: if child.type == "do_block": return child return None def _extract_elixir_module(node, keyword: str, source_bytes: bytes, filename: str, parent_symbol: Optional[Symbol]) -> Optional[Symbol]: """Extract a defmodule/defprotocol/defimpl symbol.""" arguments = _get_elixir_args(node) if arguments is None: return None # For defimpl, find `alias` (implemented module) + `for:` target if keyword == "defimpl": name = _extract_elixir_defimpl_name(arguments, source_bytes, parent_symbol) else: name = _extract_elixir_alias_name(arguments, source_bytes) if not name: return None kind = "type" if keyword == "defprotocol" else "class" if parent_symbol: qualified_name = f"{parent_symbol.qualified_name}.{name}" else: qualified_name = name # Signature: everything up to the do_block signature = _build_elixir_signature(node, source_bytes) # Moduledoc: look inside do_block do_block = _find_elixir_do_block(node) docstring = _extract_elixir_moduledoc(do_block, source_bytes) if do_block else "" return _make_elixir_symbol(node, source_bytes, filename, name, qualified_name, kind, parent_symbol, signature, docstring) def _extract_elixir_alias_name(arguments, source_bytes: bytes) -> Optional[str]: """Extract module name from an `alias` node in arguments.""" for child in arguments.children: if child.type == "alias": return source_bytes[child.start_byte:child.end_byte].decode("utf-8").strip() # Sometimes the module name is an `atom` (rare) or `identifier` if child.type in ("identifier", "atom"): return source_bytes[child.start_byte:child.end_byte].decode("utf-8").strip() return None def _extract_elixir_defimpl_name(arguments, source_bytes: bytes, parent_symbol: Optional[Symbol]) -> Optional[str]: """Build a name for defimpl: '.' or just the protocol name.""" # First child is usually the protocol alias proto_name = None for_name = None for child in arguments.children: if child.type == "alias" and proto_name is None: proto_name = source_bytes[child.start_byte:child.end_byte].decode("utf-8").strip() # `for:` keyword argument: keywords > pair > (atom "for") + alias if child.type == "keywords": for pair in child.children: if pair.type == "pair": key_node = pair.child_by_field_name("key") val_node = pair.child_by_field_name("value") if key_node and val_node: key_text = source_bytes[key_node.start_byte:key_node.end_byte].decode("utf-8").strip() if key_text in ("for", "for:"): for_name = source_bytes[val_node.start_byte:val_node.end_byte].decode("utf-8").strip() if proto_name and for_name: # e.g. Printable.Integer return f"{proto_name}.{for_name}" return proto_name def _extract_elixir_function(node, keyword: str, source_bytes: bytes, filename: str, parent_symbol: Optional[Symbol]) -> Optional[Symbol]: """Extract a def/defp/defmacro/defmacrop/defguard/defguardp symbol.""" arguments = _get_elixir_args(node) if arguments is None: return None # First named child in arguments is a `call` node (the function head) func_call = _first_named_child(arguments) if func_call is None: return None # Handle guard: `def foo(x) when is_integer(x)` — binary_operator `when` actual_call = func_call if func_call.type == "binary_operator": left = func_call.child_by_field_name("left") if left: actual_call = left name = _extract_elixir_call_name(actual_call, source_bytes) if not name: return None # Determine kind based on parent context if parent_symbol and parent_symbol.kind in ("class", "type"): kind = "method" else: kind = "function" if parent_symbol: qualified_name = f"{parent_symbol.qualified_name}.{name}" else: qualified_name = name signature = _build_elixir_signature(node, source_bytes) docstring = _extract_elixir_doc(node, source_bytes) return _make_elixir_symbol(node, source_bytes, filename, name, qualified_name, kind, parent_symbol, signature, docstring) def _extract_elixir_call_name(call_node, source_bytes: bytes) -> Optional[str]: """Extract the function name from a call node's target.""" if call_node.type == "call": target = call_node.child_by_field_name("target") if target: return source_bytes[target.start_byte:target.end_byte].decode("utf-8").strip() if call_node.type == "identifier": return source_bytes[call_node.start_byte:call_node.end_byte].decode("utf-8").strip() return None def _build_elixir_signature(node, source_bytes: bytes) -> str: """Build function/module signature: text up to the do_block.""" do_block = _find_elixir_do_block(node) if do_block: sig_bytes = source_bytes[node.start_byte:do_block.start_byte] else: sig_bytes = source_bytes[node.start_byte:node.end_byte] return sig_bytes.decode("utf-8").strip().rstrip(",").strip() def _extract_elixir_doc(node, source_bytes: bytes) -> str: """Walk backward through prev_named_sibling looking for @doc attribute.""" prev = node.prev_named_sibling while prev is not None: if prev.type == "unary_operator": attr = _get_elixir_attr_name(prev, source_bytes) if attr == "doc": inner = _first_named_child(prev) return _extract_elixir_string_arg(inner, source_bytes) if attr in _ELIXIR_SKIP_ATTRS: # Skip @spec and @impl, keep walking back prev = prev.prev_named_sibling continue # Some other attribute — stop break elif prev.type == "comment": prev = prev.prev_named_sibling continue else: break return "" def _extract_elixir_moduledoc(do_block, source_bytes: bytes) -> str: """Find @moduledoc inside a do_block and extract its string content.""" if do_block is None: return "" for child in do_block.children: if child.type == "unary_operator": if _get_elixir_attr_name(child, source_bytes) == "moduledoc": inner = _first_named_child(child) return _extract_elixir_string_arg(inner, source_bytes) return "" def _extract_elixir_string_arg(call_node, source_bytes: bytes) -> str: """Extract string content from @doc/@moduledoc argument (handles both "" and \"\"\"\"\"\").""" arguments = _get_elixir_args(call_node) if arguments is None: return "" for child in arguments.children: if child.type == "string": text = source_bytes[child.start_byte:child.end_byte].decode("utf-8") return _strip_quotes(text) # @doc false → boolean node, not a string return "" def _extract_elixir_type_attribute(node, attr_name: str, inner_call, source_bytes: bytes, filename: str, parent_symbol: Optional[Symbol]) -> Optional[Symbol]: """Extract @type/@typep/@opaque as type symbols.""" # inner_call is the `call` inside `@type name :: expr` arguments = _get_elixir_args(inner_call) if arguments is None: return None # The first named child is a `binary_operator` with `::` operator # whose left side is the type name (possibly a call for parameterized types) for child in arguments.children: if child.is_named: name = _extract_elixir_type_name(child, source_bytes) if not name: return None kind = "type" if parent_symbol: qualified_name = f"{parent_symbol.qualified_name}.{name}" else: qualified_name = name sig = _node_text(node, source_bytes) return _make_elixir_symbol(node, source_bytes, filename, name, qualified_name, kind, parent_symbol, sig) return None def _extract_elixir_type_name(type_expr_node, source_bytes: bytes) -> Optional[str]: """Extract just the name from a type expression like `name :: type` or `name(params) :: type`.""" # `binary_operator` with `::` — left side is the name if type_expr_node.type == "binary_operator": left = type_expr_node.child_by_field_name("left") if left: return _extract_elixir_type_name(left, source_bytes) # Plain `call` like `name(params)` — name is the target if type_expr_node.type == "call": target = type_expr_node.child_by_field_name("target") if target: return source_bytes[target.start_byte:target.end_byte].decode("utf-8").strip() # Plain identifier if type_expr_node.type in ("identifier", "atom"): return source_bytes[type_expr_node.start_byte:type_expr_node.end_byte].decode("utf-8").strip() return None def _disambiguate_overloads(symbols: list[Symbol]) -> list[Symbol]: """Append ordinal suffix to symbols with duplicate IDs. E.g., if two symbols have ID "file.py::foo#function", they become "file.py::foo#function~1" and "file.py::foo#function~2". """ from collections import Counter id_counts = Counter(s.id for s in symbols) # Only process IDs that appear more than once duplicated = {sid for sid, count in id_counts.items() if count > 1} if not duplicated: return symbols # Track ordinals per duplicate ID ordinals: dict[str, int] = {} result = [] for sym in symbols: if sym.id in duplicated: ordinals[sym.id] = ordinals.get(sym.id, 0) + 1 sym.id = f"{sym.id}~{ordinals[sym.id]}" result.append(sym) return result _CALLABLE_KINDS = frozenset({"function", "method"}) def _disambiguate_and_compute_complexity( symbols: list[Symbol], source_bytes: bytes ) -> list[Symbol]: """Disambiguate overloads + compute complexity in a single pass. Merges two formerly separate O(N) passes into one to reduce overhead. """ # Quick check for duplicates using a set (faster than Counter for common case) seen_ids: set[str] = set() has_duplicates = False for sym in symbols: if sym.id in seen_ids: has_duplicates = True break seen_ids.add(sym.id) # Single pass: disambiguate (if needed) + compute complexity ordinals: dict[str, int] = {} if has_duplicates: from collections import Counter id_counts = Counter(s.id for s in symbols) duplicated = {sid for sid, count in id_counts.items() if count > 1} result = [] for sym in symbols: if has_duplicates and sym.id in duplicated: ordinals[sym.id] = ordinals.get(sym.id, 0) + 1 sym.id = f"{sym.id}~{ordinals[sym.id]}" if sym.kind in _CALLABLE_KINDS and sym.byte_length > 0: body = source_bytes[sym.byte_offset:sym.byte_offset + sym.byte_length].decode("utf-8", errors="replace") sym.cyclomatic, sym.max_nesting, sym.param_count = compute_complexity(body, sym.signature) result.append(sym) return result if has_duplicates else symbols # --------------------------------------------------------------------------- # Blade template parser (regex-based; no tree-sitter grammar available) # --------------------------------------------------------------------------- _BLADE_SYMBOL_PATTERNS: list[tuple[str, str, str]] = [ ("type", r"@extends\s*\(\s*['\"](?P[^'\"]+)['\"]", "name"), ("method", r"@section\s*\(\s*['\"](?P[^'\"]+)['\"]", "name"), ("class", r"@component\s*\(\s*['\"](?P[^'\"]+)['\"]", "name"), ("function", r"@include(?:If|When|Unless|First)?\s*\(\s*['\"](?P[^'\"]+)['\"]", "name"), ("constant", r"@push\s*\(\s*['\"](?P[^'\"]+)['\"]", "name"), ("constant", r"@stack\s*\(\s*['\"](?P[^'\"]+)['\"]", "name"), ("method", r"@slot\s*\(\s*['\"](?P[^'\"]+)['\"]", "name"), ("method", r"@yield\s*\(\s*['\"](?P[^'\"]+)['\"]", "name"), ("class", r"@livewire\s*\(\s*['\"](?P[^'\"]+)['\"]", "name"), ] # --------------------------------------------------------------------------- # Verse (UEFN) — regex-based symbol extraction for Epic's Verse language # --------------------------------------------------------------------------- # # No tree-sitter grammar exists for Verse, so this parser uses regex with a # multi-pass strategy similar to the Blade parser above. # # PRIMARY USE CASE: Token-efficient lookup of UEFN API digest files. # # Epic ships Fortnite/UEFN API definitions as `.verse` digest files that are # very large (the three standard digest files total ~800KB / ~200k tokens): # # Fortnite.digest.verse 587KB 12,258 lines 3,608 symbols ~147k tokens # Verse.digest.verse 125KB 2,368 lines 622 symbols ~31k tokens # UnrealEngine.digest.verse 91KB 1,495 lines 326 symbols ~23k tokens # # Loading even one of these into an LLM context window is expensive. # With jcodemunch indexing, a typical symbol lookup returns ~94 tokens # instead of ~147,000 — a 99.9% reduction. A search returning 10 signature # matches costs ~130 tokens vs the full file's ~147k. # # ARCHITECTURE: # # Verse uses indentation-based scoping with a distinctive declaration syntax: # # name := kind(parents): # member(...):return_type # var Name:type # # Extension methods use receiver syntax: # (Param:type).MethodName():return_type # # Digest files use path-prefixed declarations for namespace qualification: # (/Fortnite.com:)UI := module: # # Decorators use @attribute syntax: # @editable # @available {MinUploadedAtFNVersion := 3800} # # The parser runs in 5 passes to handle declaration priority correctly: # Pass 1: Container definitions (module, class, interface, struct, enum, trait) # Pass 2: Extension methods — (Receiver:type).Method() syntax # Pass 3: Regular methods — indented Name(params) inside containers # Pass 4: Variables — var Name:type declarations # Pass 5: Constants — Name:type = value assignments # # IMPORTANT — Character vs byte offset handling: # # Python regex operates on decoded strings where multi-byte UTF-8 characters # (e.g., smart quotes U+2019 = 3 bytes) count as 1 character. But the # retrieval path (get_symbol_content) does binary f.seek(byte_offset), so # stored byte_offset values MUST be real byte positions — not character # positions. The char_pos_to_byte_pos() helper handles this conversion. # The Verse digest files contain multi-byte UTF-8 characters in docstrings # (smart quotes), which affects ~60% of all extracted symbols. # Shared regex fragment for Verse specifiers like , _VERSE_SPECS = r'(?:<[a-z_]+>)*' # --- Pass 1 regex: Container definitions --- # Matches: name := kind(parents): # Also: (/Fortnite.com:)name := module: _VERSE_DEF_RE = re.compile( r'^([ \t]*)' # (1) indentation — [ \t] only, NOT \s (which captures \n in MULTILINE) r'(?:\([^)]*:\))?' # optional path prefix e.g. (/Fortnite.com:) r'([\w]+)' # (2) name r'(' + _VERSE_SPECS + r')' # (3) specifiers e.g. r'\s*:=\s*' # := assignment operator r'(module|class|interface|struct|enum|trait)' # (4) kind keyword r'(' + _VERSE_SPECS + r')' # (5) kind specifiers e.g. r'(?:\(([^)]*)\))?' # (6) optional parent types e.g. (base_class) r'\s*:', # trailing colon (starts indented block) re.MULTILINE, ) # --- Pass 3 regex: Method/function members --- # Matches: Name(params):return_type # Also: (/Path:)Name(...) _VERSE_METHOD_RE = re.compile( r'^([ \t]+)' # (1) indentation — must be indented (inside a container) r'(?:\([^)]*:\))?' # optional path prefix r'([\w]+)' # (2) name r'(' + _VERSE_SPECS + r')' # (3) specifiers r'\(([^)]*)\)' # (4) parameters r'(' + _VERSE_SPECS + r')' # (5) effect specifiers e.g. r'(?::(\S+))?' # (6) optional return type r'.*$', # rest of line (may contain = external {}) re.MULTILINE, ) # --- Pass 2 regex: Extension methods --- # Matches: (Param:type).Name(params):return_type _VERSE_EXT_METHOD_RE = re.compile( r'^([ \t]*)' # (1) indentation r'\(([^)]+)\)' # (2) receiver e.g. (InCharacter:fort_character) r'\.([\w]+)' # (3) method name after dot r'(' + _VERSE_SPECS + r')' # (4) specifiers r'\(([^)]*)\)' # (5) parameters r'(' + _VERSE_SPECS + r')' # (6) effect specifiers r'(?::(\S+))?' # (7) optional return type r'.*$', re.MULTILINE, ) # --- Pass 4 regex: Variable declarations --- # Matches: var Name:type or var Name:type _VERSE_VAR_RE = re.compile( r'^([ \t]+)' # (1) indentation (must be inside container) r'var(?:<[a-z_]+>)?' # var keyword with optional specifier r'\s+' r'([\w]+)' # (2) name r'(' + _VERSE_SPECS + r')' # (3) specifiers r':([^\s=]+)' # (4) type (up to whitespace or =) r'.*$', re.MULTILINE, ) # --- Pass 5 regex: Constants/values --- # Matches: Name:type = ... # Also: (/Path:)Name:type = external {} _VERSE_CONST_RE = re.compile( r'^([ \t]+)' # (1) indentation (must be inside container) r'(?:\([^)]*:\))?' # optional path prefix r'([\w]+)' # (2) name r'(' + _VERSE_SPECS + r')' # (3) specifiers r':(\S+)' # (4) type r'\s*=\s*' # = assignment r'.*$', re.MULTILINE, ) # Enum value (simple identifier on its own line — currently unused, reserved for future) _VERSE_ENUM_VAL_RE = re.compile( r'^(\s+)' # (1) indentation r'([\w]+)' # (2) name r'\s*$', re.MULTILINE, ) # Module import path comment: # Module import path: /Something/Path _VERSE_MODULE_PATH_RE = re.compile( r'#\s*Module import path:\s*(\S+)', ) # Decorator line: @editable, @available {MinUploadedAtFNVersion := 3800} _VERSE_DECORATOR_RE = re.compile( r'^(\s*)@(\w+)\s*(.*?)$', re.MULTILINE, ) def _parse_verse_symbols(source_bytes: bytes, filename: str) -> list[Symbol]: """Extract symbols from Verse (UEFN) source files using regex. Designed for Epic's Verse API digest files (Fortnite.digest.verse, Verse.digest.verse, UnrealEngine.digest.verse). These files define the entire UEFN API surface — thousands of classes, methods, and constants — and are too large to load into an LLM context window directly (~200k tokens for all three). Indexing them with jcodemunch reduces a typical symbol lookup from ~147,000 tokens to ~94 tokens (99.9% savings). The parser runs in 5 ordered passes so earlier passes take priority over later ones via seen_ids deduplication: Pass 1: Container definitions (module, class, interface, struct, enum) Pass 2: Extension methods — (Receiver:type).Method() syntax Pass 3: Regular methods — indented Name(params) inside containers Pass 4: Variable declarations — var Name:type Pass 5: Constants — Name:type = value Parent-child relationships are determined by line-range containment: each container records its start/end line, and members are assigned to the innermost container whose line range encloses them and whose indentation is less than the member's. Args: source_bytes: Raw file content (binary). Used for byte-offset calculation and content hashing. filename: The file's path/name for symbol IDs. Returns: List of Symbol objects sorted by line number, with correct byte_offset/byte_length for binary file seeking. """ content = source_bytes.decode("utf-8", errors="replace") lines = content.splitlines() # ── Dual offset tables (char-based and byte-based) ────────────────── # # Why two tables? Python regex .start() returns CHARACTER positions in # the decoded string, but get_symbol_content() does f.seek(byte_offset) # in binary mode — it needs BYTE positions. # # For pure ASCII files these are identical. But the Verse digest files # contain multi-byte UTF-8 characters (e.g., smart quotes U+2019 = 3 # bytes \xe2\x80\x99 in docstrings). In Fortnite.digest.verse, ~60% of # symbols appear after such characters, so their char offset diverges # from their byte offset. Without this conversion, get_symbol_content() # would seek to the wrong file position and return corrupted content. char_line_starts: list[int] = [] # cumulative character offset per line byte_line_starts: list[int] = [] # cumulative byte offset per line char_off = 0 byte_off = 0 for line in lines: char_line_starts.append(char_off) byte_line_starts.append(byte_off) char_off += len(line) + 1 # +1 for \n (char count) byte_off += len(line.encode("utf-8")) + 1 # +1 for \n (byte count) def char_to_line(char_pos: int) -> int: """Map a character offset (from regex .start()) to a 1-indexed line number. Uses binary search over char_line_starts for O(log n) lookup. """ lo, hi = 0, len(char_line_starts) - 1 while lo < hi: mid = (lo + hi + 1) // 2 if char_line_starts[mid] <= char_pos: lo = mid else: hi = mid - 1 return lo + 1 # 1-indexed def char_pos_to_byte_pos(char_pos: int) -> int: """Convert a character offset (from regex .start()) to a real byte offset. This is the critical bridge between regex (which operates on decoded Python strings) and file I/O (which operates on raw bytes). The algorithm: 1. Binary-search char_line_starts to find which line char_pos is on 2. Compute how many chars into that line: char_pos - line_char_start 3. Encode just that line prefix to UTF-8 to get exact byte count 4. Return: byte_line_start + encoded_prefix_byte_length This matches tree-sitter's node.start_byte behavior for languages that have tree-sitter grammars. """ # Find the 0-based line index via binary search lo, hi = 0, len(char_line_starts) - 1 while lo < hi: mid = (lo + hi + 1) // 2 if char_line_starts[mid] <= char_pos: lo = mid else: hi = mid - 1 line_idx = lo # Encode the chars before char_pos on this line to get byte count char_into_line = char_pos - char_line_starts[line_idx] line_prefix = lines[line_idx][:char_into_line] return byte_line_starts[line_idx] + len(line_prefix.encode("utf-8")) # ── Docstring and decorator extraction ────────────────────────────── # # Verse uses # line comments for documentation and @attribute for # decorators. Both appear on lines immediately above a declaration. # We walk upward from the declaration line, skipping decorators when # gathering comments (and vice versa). def _get_preceding_comment(line_idx: int) -> str: """Gather # comment lines immediately above line_idx (0-indexed). Walks upward, collecting comment text and skipping @decorator lines that may be interspersed. Returns joined text with # prefix stripped. """ doc_lines: list[str] = [] i = line_idx - 1 while i >= 0: stripped = lines[i].strip() if stripped.startswith("#"): doc_lines.append(stripped.lstrip("# ").strip()) i -= 1 elif stripped.startswith("@"): i -= 1 # decorators can appear between comment and declaration else: break doc_lines.reverse() return "\n".join(doc_lines) def _get_decorators(line_idx: int) -> list[str]: """Gather @decorator lines immediately above line_idx (0-indexed). Walks upward, collecting decorator text and skipping # comment lines. Returns decorators in source order (top to bottom). """ decs: list[str] = [] i = line_idx - 1 while i >= 0: stripped = lines[i].strip() if stripped.startswith("@"): decs.append(stripped) i -= 1 elif stripped.startswith("#"): i -= 1 # skip comments between decorators else: break decs.reverse() return decs # ── Indentation-based block detection ─────────────────────────────── def _find_block_end(start_line_idx: int, base_indent: int) -> int: """Find the last line of an indentation block starting at start_line_idx. Verse uses indentation for scoping (like Python). A block ends when a non-blank, non-comment line appears at the base indentation level or less. Blank lines, comments, and decorator lines are skipped (they don't terminate a block). Returns: 0-indexed line number of the last line in the block. """ last = start_line_idx for i in range(start_line_idx + 1, len(lines)): stripped = lines[i].strip() if not stripped or stripped.startswith("#") or stripped.startswith("@"): continue # blank, comment, or decorator lines don't end blocks indent = len(lines[i]) - len(lines[i].lstrip()) if indent <= base_indent: break last = i return last # ── Symbol collection state ───────────────────────────────────────── symbols: list[Symbol] = [] seen_ids: set[str] = set() # prevents duplicates across passes # Containers track: (indent, qualified_name, kind_raw, start_line, end_line) # Used for parent assignment via line-range containment. This approach # correctly handles sibling containers at the same indent level — a # pure indent-only strategy would incorrectly assign members of a later # container to an earlier sibling. containers: list[tuple[int, str, str, int, int]] = [] def _find_parent(member_line_1idx: int, member_indent: int) -> "Optional[str]": """Find the innermost container enclosing this member. Uses both indentation (member must be more indented than container) and line-range containment (member line must fall within container's start..end range). When multiple containers qualify, picks the one with the greatest indentation (innermost nesting). Args: member_line_1idx: 1-indexed line number of the member. member_indent: Column indentation of the member. Returns: Qualified name of the parent container, or None if top-level. """ best = None for _indent, cname, _ckind, cstart, cend in containers: if member_indent > _indent and cstart <= member_line_1idx <= cend: if best is None or _indent > best[0]: best = (_indent, cname) return best[1] if best else None # Optional module path from header comment (e.g., # Module import path: /Verse.org/...) module_path = "" mp_match = _VERSE_MODULE_PATH_RE.search(content) if mp_match: module_path = mp_match.group(1) # ── Pass 1: Container definitions ─────────────────────────────────── # # Extracts module, class, interface, struct, enum, and trait declarations. # These are the "containers" that hold methods, vars, and constants. # Must run first so containers[] is populated for parent lookups in # later passes. # # Containers store byte_offset/byte_length spanning their FULL block # (declaration line through last indented member), so get_symbol() # returns the complete definition including all members. for m in _VERSE_DEF_RE.finditer(content): indent_str = m.group(1) indent = len(indent_str) name = m.group(2) specs = m.group(3) kind_raw = m.group(4) kind_specs = m.group(5) parents = m.group(6) or "" # Use group(2) (the name) for line lookup — group(1) is indentation, # and m.start(0) could include characters from a prior line due to # ^ anchor behavior in MULTILINE mode with [ \t]* matching empty. line_idx = char_to_line(m.start(2)) - 1 # 0-indexed end_line_idx = _find_block_end(line_idx, indent) # Map Verse declaration kinds to jcodemunch symbol kinds. # Modules map to "class" because they act as namespaces/containers. kind_map = { "module": "class", "class": "class", "interface": "type", "struct": "type", "enum": "type", "trait": "type", } kind = kind_map.get(kind_raw, "type") sig_parts = [f"{name}{specs} := {kind_raw}{kind_specs}"] if parents: sig_parts.append(f"({parents})") signature = "".join(sig_parts) docstring = _get_preceding_comment(line_idx) decorators = _get_decorators(line_idx) parent_name = _find_parent(line_idx + 1, indent) qualified = f"{parent_name}.{name}" if parent_name else name sym_id = make_symbol_id(filename, qualified, kind) if sym_id not in seen_ids: seen_ids.add(sym_id) match_bytes = m.group(0).encode("utf-8") # Compute byte range for the entire container block. # block_byte_start = byte position of the declaration line. # block_byte_end = end of the last indented member line. block_byte_start = char_pos_to_byte_pos(m.start()) if end_line_idx < len(byte_line_starts): block_byte_end = byte_line_starts[end_line_idx] + len(lines[end_line_idx].encode("utf-8")) else: block_byte_end = block_byte_start + len(match_bytes) symbols.append(Symbol( id=sym_id, file=filename, name=name, qualified_name=qualified, kind=kind, language="verse", signature=signature, docstring=docstring, decorators=decorators, parent=make_symbol_id(filename, parent_name, "class") if parent_name else None, line=line_idx + 1, end_line=end_line_idx + 1, byte_offset=block_byte_start, byte_length=block_byte_end - block_byte_start, content_hash=compute_content_hash(source_bytes[block_byte_start:block_byte_end]), )) # Register container for parent lookups in passes 2-5 containers.append((indent, qualified, kind_raw, line_idx + 1, end_line_idx + 1)) # ── Pass 2: Extension methods ─────────────────────────────────────── # # Verse extension methods use receiver syntax: # (InPlayer:player).GetScore():int # # These are matched separately because they have a distinctive # (Receiver:type).Name pattern that doesn't overlap with regular methods. for m in _VERSE_EXT_METHOD_RE.finditer(content): indent_str = m.group(1) indent = len(indent_str) receiver = m.group(2) name = m.group(3) specs = m.group(4) params = m.group(5) effects = m.group(6) ret_type = m.group(7) or "" line_idx = char_to_line(m.start(2)) - 1 sig = f"({receiver}).{name}{specs}({params}){effects}" if ret_type: sig += f":{ret_type}" # Qualified name uses the receiver type (e.g., player.GetScore) recv_type = receiver.split(":")[-1].strip() if ":" in receiver else receiver qualified = f"{recv_type}.{name}" # Extension methods can appear inside module blocks parent_name = _find_parent(line_idx + 1, indent) if parent_name: qualified = f"{parent_name}.{name}" sym_id = make_symbol_id(filename, qualified, "method") if sym_id not in seen_ids: seen_ids.add(sym_id) docstring = _get_preceding_comment(line_idx) decorators = _get_decorators(line_idx) match_bytes = m.group(0).encode("utf-8") symbols.append(Symbol( id=sym_id, file=filename, name=name, qualified_name=qualified, kind="method", language="verse", signature=sig, docstring=docstring, decorators=decorators, parent=make_symbol_id(filename, parent_name, "class") if parent_name else None, line=line_idx + 1, end_line=line_idx + 1, byte_offset=char_pos_to_byte_pos(m.start()), byte_length=len(match_bytes), content_hash=compute_content_hash(match_bytes), )) # ── Pass 3: Regular methods inside containers ─────────────────────── # # Matches indented Name(params) declarations that weren't already # captured as container definitions (Pass 1) or extension methods # (Pass 2). Requires a parent container — top-level functions with # params would be unusual in digest files and are skipped. for m in _VERSE_METHOD_RE.finditer(content): indent_str = m.group(1) indent = len(indent_str) name = m.group(2) specs = m.group(3) params = m.group(4) effects = m.group(5) ret_type = m.group(6) or "" line_idx = char_to_line(m.start(2)) - 1 # Guard: skip lines already handled by other passes full_line = lines[line_idx].strip() if line_idx < len(lines) else "" if ":=" in full_line: continue # definition line (Pass 1) if full_line.startswith("var"): continue # variable declaration (Pass 4) parent_name = _find_parent(line_idx + 1, indent) if not parent_name: continue # methods must be inside a container qualified = f"{parent_name}.{name}" kind = "method" sym_id = make_symbol_id(filename, qualified, kind) if sym_id not in seen_ids: seen_ids.add(sym_id) sig = f"{name}{specs}({params}){effects}" if ret_type: sig += f":{ret_type}" docstring = _get_preceding_comment(line_idx) decorators = _get_decorators(line_idx) match_bytes = m.group(0).encode("utf-8") symbols.append(Symbol( id=sym_id, file=filename, name=name, qualified_name=qualified, kind=kind, language="verse", signature=sig, docstring=docstring, decorators=decorators, parent=make_symbol_id(filename, parent_name, "class") if parent_name else None, line=line_idx + 1, end_line=line_idx + 1, byte_offset=char_pos_to_byte_pos(m.start()), byte_length=len(match_bytes), content_hash=compute_content_hash(match_bytes), )) # ── Pass 4: Variable declarations ─────────────────────────────────── # # Matches: var Name:type # Stored as "constant" kind (jcodemunch doesn't distinguish var/const). for m in _VERSE_VAR_RE.finditer(content): indent_str = m.group(1) indent = len(indent_str) name = m.group(2) specs = m.group(3) var_type = m.group(4) line_idx = char_to_line(m.start(2)) - 1 parent_name = _find_parent(line_idx + 1, indent) qualified = f"{parent_name}.{name}" if parent_name else name sym_id = make_symbol_id(filename, qualified, "constant") if sym_id not in seen_ids: seen_ids.add(sym_id) sig = f"var {name}{specs}:{var_type}" docstring = _get_preceding_comment(line_idx) match_bytes = m.group(0).encode("utf-8") symbols.append(Symbol( id=sym_id, file=filename, name=name, qualified_name=qualified, kind="constant", language="verse", signature=sig, docstring=docstring, parent=make_symbol_id(filename, parent_name, "class") if parent_name else None, line=line_idx + 1, end_line=line_idx + 1, byte_offset=char_pos_to_byte_pos(m.start()), byte_length=len(match_bytes), content_hash=compute_content_hash(match_bytes), )) # ── Pass 5: Constants and value declarations ──────────────────────── # # Matches: Name:type = external {} # This is the most common pattern in digest files for API surface # declarations. Runs last so vars (Pass 4) and definitions (Pass 1) # take priority via seen_ids. for m in _VERSE_CONST_RE.finditer(content): indent_str = m.group(1) indent = len(indent_str) name = m.group(2) specs = m.group(3) const_type = m.group(4) line_idx = char_to_line(m.start(2)) - 1 # Guard: skip lines handled by earlier passes full_line = lines[line_idx].strip() if line_idx < len(lines) else "" if full_line.startswith("var"): continue # var declaration (Pass 4) if ":=" in full_line: continue # definition line (Pass 1) parent_name = _find_parent(line_idx + 1, indent) qualified = f"{parent_name}.{name}" if parent_name else name sym_id = make_symbol_id(filename, qualified, "constant") if sym_id not in seen_ids: seen_ids.add(sym_id) sig = f"{name}{specs}:{const_type}" docstring = _get_preceding_comment(line_idx) match_bytes = m.group(0).encode("utf-8") symbols.append(Symbol( id=sym_id, file=filename, name=name, qualified_name=qualified, kind="constant", language="verse", signature=sig, docstring=docstring, parent=make_symbol_id(filename, parent_name, "class") if parent_name else None, line=line_idx + 1, end_line=line_idx + 1, byte_offset=char_pos_to_byte_pos(m.start()), byte_length=len(match_bytes), content_hash=compute_content_hash(match_bytes), )) symbols.sort(key=lambda s: s.line) return symbols _BLADE_COMPILED: list[tuple[str, re.Pattern, str]] = [ (kind, re.compile(pattern, re.IGNORECASE), group) for kind, pattern, group in _BLADE_SYMBOL_PATTERNS ] def _parse_blade_symbols(source_bytes: bytes, filename: str) -> list[Symbol]: """Extract Blade template symbols using regex. Scans for directives that define meaningful structural elements: @extends, @section, @component, @include*, @push, @stack, @slot, @yield, @livewire. No tree-sitter grammar exists for Blade. """ content = source_bytes.decode("utf-8", errors="replace") lines = content.splitlines() line_start_offsets: list[int] = [] offset = 0 for line in lines: line_start_offsets.append(offset) offset += len(line.encode("utf-8")) + 1 def byte_to_line(byte_pos: int) -> int: lo, hi = 0, len(line_start_offsets) - 1 while lo < hi: mid = (lo + hi + 1) // 2 if line_start_offsets[mid] <= byte_pos: lo = mid else: hi = mid - 1 return lo + 1 symbols: list[Symbol] = [] seen: set[tuple[str, str]] = set() for kind, pattern, group in _BLADE_COMPILED: for m in pattern.finditer(content): name = m.group(group) key = (kind, name) if key in seen: continue seen.add(key) line_no = byte_to_line(m.start()) directive_text = m.group(0) sym_bytes = directive_text.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, name, kind), file=filename, name=name, qualified_name=name, kind=kind, language="blade", signature=directive_text, docstring="", parent=None, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) symbols.sort(key=lambda s: s.line) return symbols # --------------------------------------------------------------------------- # AL (Business Central) parser (regex-based; no tree-sitter grammar available) # --------------------------------------------------------------------------- _AL_OBJECT_TYPES_TYPE = frozenset({"enum", "interface"}) # Parent-type filter sets for child-symbol passes _AL_ENUM_PARENTS = frozenset({"enum", "enumextension"}) _AL_PAGE_ACTION_PARENTS = frozenset({"page", "pageextension"}) _AL_KEY_PARENTS = frozenset({"table", "tableextension"}) _AL_COLUMN_PARENTS = frozenset({"report", "query", "reportextension"}) _AL_FIELDGROUP_PARENTS = frozenset({"table", "tableextension"}) _AL_DATAITEM_PARENTS = frozenset({"report", "query", "reportextension"}) _AL_XMLPORT_PARENTS = frozenset({"xmlport"}) _AL_EVENT_PARENTS = frozenset({"controladdin"}) _AL_PAGE_FIELD_PARENTS = frozenset({"page", "pageextension"}) _AL_OBJECT_RE = re.compile( r"^(?Ptable|page|codeunit|report|xmlport|query|enum|interface|" r"controladdin|profile|pagecustomization|entitlement|permissionset|" r"permissionsetextension|tableextension|pageextension|enumextension|reportextension)" r"\s+(?:(?P\d+)\s+)?(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))" r"(?:\s+extends\s+(?:\"[^\"]+\"|[A-Za-z_]\w*))?", re.MULTILINE | re.IGNORECASE, ) _AL_PROCEDURE_RE = re.compile( r"(?Plocal|internal|protected)\s+procedure\s+(?P[A-Za-z_]\w*)\s*\((?P[^)]*)\)(?:\s*:\s*(?P[^;\n{]+))?" r"|procedure\s+(?P[A-Za-z_]\w*)\s*\((?P[^)]*)\)(?:\s*:\s*(?P[^;\n{]+))?", re.MULTILINE | re.IGNORECASE, ) _AL_TRIGGER_RE = re.compile( r"trigger\s+(?P[A-Za-z_]\w*)\s*\(", re.MULTILINE | re.IGNORECASE, ) _AL_FIELD_RE = re.compile( r"field\s*\(\s*(?P\d+)\s*;\s*(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))\s*;\s*(?P[^)]+)\)", re.MULTILINE | re.IGNORECASE, ) _AL_ENUM_VALUE_RE = re.compile( r"value\s*\(\s*(?P\d+)\s*;\s*(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))\s*\)", re.MULTILINE | re.IGNORECASE, ) _AL_ACTION_RE = re.compile( r"action\s*\(\s*(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))\s*\)", re.MULTILINE | re.IGNORECASE, ) _AL_KEY_RE = re.compile( r"key\s*\(\s*(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))\s*;\s*(?P[^)]+)\)", re.MULTILINE | re.IGNORECASE, ) _AL_COLUMN_RE = re.compile( r"column\s*\(\s*(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))\s*;\s*(?P[^)]+)\)", re.MULTILINE | re.IGNORECASE, ) _AL_FIELDGROUP_RE = re.compile( r"fieldgroup\s*\(\s*(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))\s*;\s*(?P[^)]+)\)", re.MULTILINE | re.IGNORECASE, ) _AL_DATAITEM_RE = re.compile( r"dataitem\s*\(\s*(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))\s*;\s*(?P[^)]+)\)", re.MULTILINE | re.IGNORECASE, ) _AL_XMLPORT_ELEMENT_RE = re.compile( r"(?Ptableelement|textelement|fieldelement|fieldattribute)\s*\(\s*(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))\s*(?:;\s*(?P[^)]+))?\)", re.MULTILINE | re.IGNORECASE, ) _AL_EVENT_RE = re.compile( r"event\s+(?P[A-Za-z_]\w*)\s*\((?P[^)]*)\)", re.MULTILINE | re.IGNORECASE, ) _AL_PAGE_FIELD_RE = re.compile( r"field\s*\(\s*(?:\"(?P[^\"]+)\"|(?P[A-Za-z_]\w*))\s*;\s*(?P[^)]+)\)", re.MULTILINE | re.IGNORECASE, ) _AL_VAR_RE = re.compile( r"^\s+(?P[A-Za-z_]\w*)\s*:\s*(?PRecord\s+\"[^\"]+\"|[A-Za-z_][\w\[\]\s]*?)(?:\s+temporary)?\s*;", re.MULTILINE, ) _AL_ATTR_RE = re.compile( r"\[(?P[A-Za-z_]\w*)\s*(?:\([^]]*\))?\]", ) _AL_DOC_RE = re.compile( r"///\s*(?:)?\s*(?P.*?)(?:)?\s*$", ) def _parse_al_symbols(source_bytes: bytes, filename: str) -> list[Symbol]: """Extract AL (Business Central) symbols using regex. Scans for object declarations (table, page, codeunit, etc.), procedures, triggers, table fields, enum values, page actions, keys, columns, fieldgroups, dataitems, xmlport elements, controladdin events, page layout fields, and variable declarations. """ content = source_bytes.decode("utf-8", errors="replace") lines = content.splitlines() # Build line offset table line_start_offsets: list[int] = [] offset = 0 for line in lines: line_start_offsets.append(offset) offset += len(line.encode("utf-8")) + 1 def byte_to_line(byte_pos: int) -> int: lo, hi = 0, len(line_start_offsets) - 1 while lo < hi: mid = (lo + hi + 1) // 2 if line_start_offsets[mid] <= byte_pos: lo = mid else: hi = mid - 1 return lo + 1 # Pass 1: find top-level objects and their byte ranges objects: list[tuple[str, str, int, int, str]] = [] # (name, kind, start, end, objtype) obj_matches = list(_AL_OBJECT_RE.finditer(content)) for i, m in enumerate(obj_matches): objtype = m.group("objtype").lower() name = m.group("qname") or m.group("iname") if not name: continue kind = "type" if objtype in _AL_OBJECT_TYPES_TYPE else "class" start = m.start() end = obj_matches[i + 1].start() if i + 1 < len(obj_matches) else len(content) objects.append((name, kind, start, end, objtype)) symbols: list[Symbol] = [] # Emit object symbols for name, kind, start, end, _objtype in objects: line_no = byte_to_line(start) sig_end = content.find("\n", start) if sig_end == -1: sig_end = len(content) signature = content[start:sig_end].strip() sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, name, kind), file=filename, name=name, qualified_name=name, kind=kind, language="al", signature=signature, docstring="", parent=None, line=line_no, end_line=line_no, byte_offset=start, byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) def _find_parent(pos: int) -> Optional[tuple[str, str, str]]: """Find the parent object for a given byte position. Returns (name, symbol_id, objtype) or None. """ for name, kind, start, end, objtype in objects: if start <= pos < end: return (name, make_symbol_id(filename, name, kind), objtype) return None def _extract_al_docstring(pos: int) -> str: """Extract doc comment preceding a byte position. Checks for /// XML doc comments first, then falls back to // inline comments. """ line_idx = byte_to_line(pos) - 1 # 0-indexed # First pass: look for /// XML doc comments doc_lines: list[str] = [] idx = line_idx - 1 while idx >= 0: stripped = lines[idx].strip() if stripped.startswith("///"): doc_lines.insert(0, stripped[3:].strip()) idx -= 1 elif _AL_ATTR_RE.match(stripped): # Skip past attribute lines to find doc comments above them idx -= 1 else: break if doc_lines: text = " ".join(doc_lines) text = text.replace("", "").replace("", "").strip() return text # Fallback: look for // inline comments idx = line_idx - 1 while idx >= 0: stripped = lines[idx].strip() if stripped.startswith("//") and not stripped.startswith("///"): doc_lines.insert(0, stripped[2:].strip()) idx -= 1 elif _AL_ATTR_RE.match(stripped): idx -= 1 else: break if doc_lines: return " ".join(doc_lines) return "" def _extract_al_decorators(pos: int) -> list[str]: """Extract [Attribute(...)] lines preceding a byte position.""" line_idx = byte_to_line(pos) - 1 # 0-indexed attrs: list[str] = [] idx = line_idx - 1 while idx >= 0: stripped = lines[idx].strip() if _AL_ATTR_RE.match(stripped): attrs.insert(0, stripped) idx -= 1 elif stripped.startswith("///") or stripped.startswith("//"): # Skip past doc comment lines idx -= 1 else: break return attrs # Pass 2: find procedures for m in _AL_PROCEDURE_RE.finditer(content): access = m.group("access") or "" name = m.group("name") or m.group("name2") params = m.group("params") or m.group("params2") or "" ret = m.group("return") or m.group("return2") or "" if not name: continue parent_info = _find_parent(m.start()) parent_name = parent_info[0] if parent_info else None parent_id = parent_info[1] if parent_info else None qualified_name = f"{parent_name}.{name}" if parent_name else name sig_parts = [] if access: sig_parts.append(access) sig_parts.append(f"procedure {name}({params.strip()})") if ret: sig_parts.append(f": {ret.strip()}") signature = " ".join(sig_parts) docstring = _extract_al_docstring(m.start()) decorators = _extract_al_decorators(m.start()) line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "method"), file=filename, name=name, qualified_name=qualified_name, kind="method", language="al", signature=signature, docstring=docstring, decorators=decorators, parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 3: find triggers for m in _AL_TRIGGER_RE.finditer(content): name = m.group("name") parent_info = _find_parent(m.start()) parent_name = parent_info[0] if parent_info else None parent_id = parent_info[1] if parent_info else None qualified_name = f"{parent_name}.{name}" if parent_name else name signature = f"trigger {name}()" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "method"), file=filename, name=name, qualified_name=qualified_name, kind="method", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 4: find fields (only in table/tableextension objects) for m in _AL_FIELD_RE.finditer(content): name = m.group("qname") or m.group("iname") field_type = m.group("type").strip() if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_KEY_PARENTS: # table/tableextension continue qualified_name = f"{parent_name}.{name}" signature = f"field({m.group('id')}; {name}; {field_type})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "constant"), file=filename, name=name, qualified_name=qualified_name, kind="constant", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 5: find enum values (only in enum/enumextension objects) for m in _AL_ENUM_VALUE_RE.finditer(content): name = m.group("qname") or m.group("iname") if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_ENUM_PARENTS: continue qualified_name = f"{parent_name}.{name}" signature = f"value({m.group('id')}; {name})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "constant"), file=filename, name=name, qualified_name=qualified_name, kind="constant", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 6: find page actions (only in page/pageextension objects) for m in _AL_ACTION_RE.finditer(content): name = m.group("qname") or m.group("iname") if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_PAGE_ACTION_PARENTS: continue qualified_name = f"{parent_name}.{name}" signature = f"action({name})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "function"), file=filename, name=name, qualified_name=qualified_name, kind="function", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 7: find keys (only in table/tableextension objects) for m in _AL_KEY_RE.finditer(content): name = m.group("qname") or m.group("iname") if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_KEY_PARENTS: continue columns = m.group("columns").strip() qualified_name = f"{parent_name}.{name}" signature = f"key({name}; {columns})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "constant"), file=filename, name=name, qualified_name=qualified_name, kind="constant", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 8: find report/query columns (only in report/query/reportextension) for m in _AL_COLUMN_RE.finditer(content): name = m.group("qname") or m.group("iname") if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_COLUMN_PARENTS: continue source = m.group("source").strip() qualified_name = f"{parent_name}.{name}" signature = f"column({name}; {source})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "constant"), file=filename, name=name, qualified_name=qualified_name, kind="constant", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 9: find fieldgroups (only in table/tableextension objects) for m in _AL_FIELDGROUP_RE.finditer(content): name = m.group("qname") or m.group("iname") if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_FIELDGROUP_PARENTS: continue fields = m.group("fields").strip() qualified_name = f"{parent_name}.{name}" signature = f"fieldgroup({name}; {fields})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "constant"), file=filename, name=name, qualified_name=qualified_name, kind="constant", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 10: find dataitems (only in report/query/reportextension) for m in _AL_DATAITEM_RE.finditer(content): name = m.group("qname") or m.group("iname") if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_DATAITEM_PARENTS: continue source = m.group("source").strip() qualified_name = f"{parent_name}.{name}" signature = f"dataitem({name}; {source})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "type"), file=filename, name=name, qualified_name=qualified_name, kind="type", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 11: find xmlport elements (only in xmlport objects) for m in _AL_XMLPORT_ELEMENT_RE.finditer(content): name = m.group("qname") or m.group("iname") if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_XMLPORT_PARENTS: continue elemtype = m.group("elemtype").lower() source = m.group("source") qualified_name = f"{parent_name}.{name}" if source: signature = f"{elemtype}({name}; {source.strip()})" else: signature = f"{elemtype}({name})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "type"), file=filename, name=name, qualified_name=qualified_name, kind="type", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 12: find controladdin events (only in controladdin objects) for m in _AL_EVENT_RE.finditer(content): name = m.group("name") if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_EVENT_PARENTS: continue params = m.group("params").strip() qualified_name = f"{parent_name}.{name}" signature = f"event {name}({params})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "method"), file=filename, name=name, qualified_name=qualified_name, kind="method", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 13: find page layout fields (only in page/pageextension) # These use field(Name; Source) without a numeric ID, unlike table fields for m in _AL_PAGE_FIELD_RE.finditer(content): name = m.group("qname") or m.group("iname") if not name: continue parent_info = _find_parent(m.start()) if parent_info is None: continue parent_name, parent_id, parent_objtype = parent_info if parent_objtype not in _AL_PAGE_FIELD_PARENTS: continue # Skip if this position was already matched by the table field regex (has numeric ID) line_text = lines[byte_to_line(m.start()) - 1] if byte_to_line(m.start()) <= len(lines) else "" if _AL_FIELD_RE.search(line_text): continue source = m.group("source").strip() qualified_name = f"{parent_name}.{name}" signature = f"field({name}; {source})" line_no = byte_to_line(m.start()) sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "constant"), file=filename, name=name, qualified_name=qualified_name, kind="constant", language="al", signature=signature, docstring="", parent=parent_id, line=line_no, end_line=line_no, byte_offset=m.start(), byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # Pass 14: find variable declarations (inside var sections) _in_var = False for i, line in enumerate(lines): stripped = line.strip() if stripped.lower() == "var": _in_var = True continue if _in_var: if stripped.lower().startswith(("begin", "procedure ", "trigger ", "local ", "internal ", "protected ")): _in_var = False continue if not stripped or stripped.startswith("//") or stripped.startswith("{"): continue vm = _AL_VAR_RE.match(line) if vm: vname = vm.group("name") vtype = vm.group("type").strip() # Find parent object for this line line_byte = line_start_offsets[i] if i < len(line_start_offsets) else 0 parent_info = _find_parent(line_byte) parent_name = parent_info[0] if parent_info else None parent_id = parent_info[1] if parent_info else None qualified_name = f"{parent_name}.{vname}" if parent_name else vname signature = f"{vname}: {vtype}" sym_bytes = signature.encode("utf-8") symbols.append(Symbol( id=make_symbol_id(filename, qualified_name, "constant"), file=filename, name=vname, qualified_name=qualified_name, kind="constant", language="al", signature=signature, docstring="", parent=parent_id, line=i + 1, end_line=i + 1, byte_offset=line_byte, byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) symbols.sort(key=lambda s: s.line) return symbols # Nix custom symbol extractor # --------------------------------------------------------------------------- def _parse_nix_symbols(source_bytes: bytes, filename: str) -> list[Symbol]: """Extract symbols from Nix expression files. Nix is a pure expression language; all definitions are `binding` nodes inside `binding_set` children of `let_expression` or `attrset_expression`. We walk up to MAX_DEPTH levels deep and extract bindings whose attrpath is a single identifier (i.e. not a dotted path like `environment.packages`). Bindings whose RHS is a `function_expression` are classified as functions; all others are classified as constants. """ from tree_sitter_language_pack import get_parser as _get_parser parser = _get_parser("nix") tree = parser.parse(source_bytes) symbols: list[Symbol] = [] _walk_nix_bindings(tree.root_node, source_bytes, filename, symbols, depth=0) symbols.sort(key=lambda s: s.line) return symbols def _walk_nix_bindings(node, source_bytes: bytes, filename: str, symbols: list, depth: int) -> None: """Recursively walk Nix AST, extracting bindings as symbols.""" MAX_DEPTH = 4 if depth > MAX_DEPTH: return for child in node.children: if child.type == "binding": _extract_nix_binding(child, source_bytes, filename, symbols) elif child.type in ("binding_set", "let_expression", "attrset_expression", "source_code"): _walk_nix_bindings(child, source_bytes, filename, symbols, depth + 1) def _extract_nix_binding(node, source_bytes: bytes, filename: str, symbols: list) -> None: """Extract a single Nix binding as a Symbol if it has a simple (non-dotted) name.""" attrpath_node = node.child_by_field_name("attrpath") expr_node = node.child_by_field_name("expression") if not attrpath_node or not expr_node: return # Only extract simple identifiers, skip dotted paths like `meta.description` name_children = [c for c in attrpath_node.children if c.is_named] if len(name_children) != 1 or name_children[0].type != "identifier": return name = source_bytes[name_children[0].start_byte:name_children[0].end_byte].decode("utf-8") kind = "function" if expr_node.type == "function_expression" else "constant" # Signature: binding up to (not including) the expression, + first line of RHS eq_end = expr_node.start_byte lhs = source_bytes[node.start_byte:eq_end].decode("utf-8").strip().rstrip("=").strip() rhs_first = source_bytes[expr_node.start_byte:expr_node.end_byte].decode("utf-8").splitlines()[0].strip() if len(rhs_first) > 60: rhs_first = rhs_first[:60] + "..." signature = f"{lhs} = {rhs_first}" # Docstring: preceding comment sibling. # In Nix, comments before the first binding in a binding_set appear as # siblings of the binding_set itself (inside let_expression), not of the # binding, so we also check the parent node's preceding sibling. docstring = "" comment_lines = [] prev = node.prev_named_sibling while prev and prev.type == "comment": comment_lines.insert(0, source_bytes[prev.start_byte:prev.end_byte].decode("utf-8")) prev = prev.prev_named_sibling if not comment_lines and node.prev_named_sibling is None and node.parent: prev = node.parent.prev_named_sibling while prev and prev.type == "comment": comment_lines.insert(0, source_bytes[prev.start_byte:prev.end_byte].decode("utf-8")) prev = prev.prev_named_sibling if comment_lines: docstring = _clean_comment_markers("\n".join(comment_lines)) sym_bytes = source_bytes[node.start_byte:node.end_byte] row, _ = node.start_point end_row, _ = node.end_point symbols.append(Symbol( id=make_symbol_id(filename, name, kind), file=filename, name=name, qualified_name=name, kind=kind, language="nix", signature=signature, docstring=docstring, parent=None, line=row + 1, end_line=end_row + 1, byte_offset=node.start_byte, byte_length=len(sym_bytes), content_hash=compute_content_hash(sym_bytes), )) # --------------------------------------------------------------------------- # Vue SFC custom symbol extractor # --------------------------------------------------------------------------- def _parse_vue_symbols(source_bytes: bytes, filename: str) -> list[Symbol]: """Extract symbols from Vue Single-File Components (.vue). Handles both Composition API (", re.IGNORECASE | re.DOTALL) _RAZOR_STYLE_RE = re.compile(r"]*)>(.*?)", re.IGNORECASE | re.DOTALL) _RAZOR_ID_RE = re.compile(r"""\bid\s*=\s*["']([^"'<>]+)["']""", re.IGNORECASE) _RAZOR_SCRIPT_SRC_RE = re.compile(r"""\bsrc\s*=\s*["']([^"'<>]+)["']""", re.IGNORECASE) _RAZOR_CODE_BLOCK_RE = re.compile(r"@(?:functions|code)\s*\{", re.IGNORECASE) # Blazor-specific directives (@page route, @inject Type Name) _RAZOR_PAGE_RE = re.compile(r'^@page\s+"([^"]+)"', re.MULTILINE) _RAZOR_INJECT_RE = re.compile(r'^@inject\s+(\S+)\s+(\w+)', re.MULTILINE) def _parse_razor_symbols(source_bytes: bytes, filename: str) -> list[Symbol]: """Extract symbols from Razor (.cshtml / .razor) templates. Strategy: - Synthetic view/component symbol from filename - HTML ids as constant symbols -