"""Identifier-aware tokenization shared by indexing and ranking."""

from __future__ import annotations

import re
from collections import Counter
from functools import lru_cache

STOPWORDS = frozenset(
    """
a an and any are as at be been but by can could do does doing done for from get gets
has have how i if in into is it its me my no not of on or our so than that the their
them then there these they this those to too us was we were what when where which who
whom why will with would you your
class const def else elif export extern false final for foreach func function if impl
import interface let module new null nil none package private protected public
return self static struct super this throw true try type typeof use using val var void
while yield
code file files line lines value values
""".split()
)

_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
_CAMEL_PART = re.compile(r"[A-Z]+(?![a-z])|[A-Z][a-z0-9]*|[a-z0-9]+")
_PATH_SEP = re.compile(r"[\\/]+")


def split_identifier(identifier: str) -> list[str]:
    parts: list[str] = []
    for chunk in identifier.split("_"):
        parts.extend(_CAMEL_PART.findall(chunk))
    return [part.lower() for part in parts if part]


@lru_cache(maxsize=100_000)
def _expand(identifier: str) -> tuple[str, ...]:
    pieces = split_identifier(identifier)
    if len(pieces) > 1:
        pieces = [identifier.lower(), *pieces]
    return tuple(token for token in pieces if len(token) > 1 and token not in STOPWORDS)


def tokenize(text: str) -> list[str]:
    """Lowercased tokens: whole identifiers plus their camel/snake components."""
    out: list[str] = []
    for identifier in _IDENTIFIER.findall(text):
        out.extend(_expand(identifier))
    return out


def identifiers(text: str) -> list[str]:
    return _IDENTIFIER.findall(text)


def path_tokens(rel_path: str) -> list[str]:
    return tokenize(_PATH_SEP.sub(" ", rel_path).replace(".", " "))


DIGEST_LIMIT = 96
DIGEST_HEADER_LINES = 30


def digest(text: str, limit: int = DIGEST_LIMIT) -> list[str]:
    """Body tokens kept in the graph so ranking can preselect files without reading them.

    Header tokens outrank frequency: what a file says it is about in its first
    lines predicts queries better than whatever identifier it repeats most.
    """
    counts = Counter(token for token in tokenize(text) if len(token) >= 3)
    if not counts:
        return []
    header = {
        token
        for token in tokenize("\n".join(text.split("\n")[:DIGEST_HEADER_LINES]))
        if len(token) >= 3
    }
    ranked = sorted(counts.items(), key=lambda item: (item[0] in header, item[1]), reverse=True)
    return [token for token, _ in ranked[:limit]]