"""Discovery of user-authored custom evaluators (LLM-judge and code-only).

Two phases, both lazy per spec FR-004 ("load custom evaluators only when
referenced"):

* :func:`enumerate_custom_evaluator_folders` — one cheap directory scan of
  ``<root>/custom-evaluators/<name>/``. Validates structure only (no imports,
  no scanning) and returns ``(complete, incomplete)`` name→path maps, where
  *complete* folders contain a ``<name>.py`` (the only strictly required file)
  and *incomplete* folders are missing it. Lets the resolver give a targeted
  "missing required file" error instead of "unknown evaluator".
* :func:`load_custom_evaluator` — imports one referenced evaluator's ``.py``,
  resolves its top-level class, and runs the security scanner.

A custom evaluator's *kind* is auto-detected from the presence of a
``<name>.prompty`` file alongside ``<name>.py``:

* ``<name>.py`` **and** ``<name>.prompty`` → **LLM-judge** evaluator. The
  customer owns prompty-loading and JSON-parsing in the ``.py``; it is
  constructed with ``model_config``.
* ``<name>.py`` only → **code-only (non-LLM)** evaluator. No LLM/Azure config is
  involved; it is constructed without ``model_config`` and performs a pure
  code-based check. As a guard against a forgotten prompty, a ``.py``-only
  wrapper whose code looks LLM-shaped (references ``load_flow`` / ``promptflow``
  / ``model_config`` / ``.prompty``) emits a non-blocking warning at load time.

See FR-001..FR-005, FR-029.
"""

from __future__ import annotations

import ast
import importlib.util
import inspect
import re
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Optional, Type

from cli_logging.cli_logger import emit_structured_log
from cli_logging.logging_utils import Operation
from custom_evaluators.security_scanner import (
    scan_prompty_file,
    scan_python_file,
)

# Folder at the user's project root (cwd) where custom evaluators live.
DEFAULT_CUSTOM_EVALUATORS_DIRNAME = "custom-evaluators"

# Valid identifier pattern, matches the schema's EvaluatorMap.propertyNames.
_VALID_NAME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9_]*$")


@dataclass(frozen=True)
class CustomEvaluatorSpec:
    """How to instantiate one loaded custom evaluator.

    ``is_llm`` records the auto-detected kind: ``True`` for an LLM-judge
    evaluator (a ``<name>.prompty`` is present and ``prompty_path`` is set),
    ``False`` for a code-only (non-LLM) evaluator (``prompty_path`` is ``None``).
    """
    name: str
    py_path: Path
    user_class: Type
    is_llm: bool
    prompty_path: Optional[Path] = None


class CustomEvaluatorDiscoveryError(ValueError):
    """Raised when a referenced custom evaluator cannot be loaded.

    Examples: a required file is missing, the ``.py`` module has a syntax
    error, fails to import (missing dependency), or defines no top-level
    class. The evaluation runner catches this and reports it as an inline
    ``error`` result per FR-026.
    """


def enumerate_custom_evaluator_folders(
    root_dir: Optional[Path] = None,
) -> tuple[Dict[str, Path], Dict[str, Path]]:
    """Scan ``custom-evaluators/`` once for structurally valid evaluator folders.

    Structure-only check (valid identifier name, ``<name>.py`` present); never
    imports user code or runs the scanner (FR-004) — that happens lazily in
    :func:`load_custom_evaluator`.

    A folder's *kind* (LLM-judge vs code-only) is NOT decided here — it is
    derived from the presence of a ``<name>.prompty`` file at load time. The
    only strictly required file is ``<name>.py``.

    Args:
        root_dir: Project root to scan. Defaults to the current directory.

    Returns:
        A ``(complete, incomplete)`` pair of name→folder maps. *complete* has a
        ``<name>.py`` (with an optional sibling ``<name>.prompty`` selecting the
        LLM-judge kind); *incomplete* is missing ``<name>.py`` (lets the resolver
        report a targeted "missing required file" error). Non-identifier, hidden,
        and dunder folders appear in neither. Both are ``{}`` when
        ``custom-evaluators/`` is absent (the common case).
    """
    base = Path(root_dir) if root_dir is not None else Path.cwd()
    custom_dir = base / DEFAULT_CUSTOM_EVALUATORS_DIRNAME
    if not custom_dir.is_dir():
        return {}, {}

    complete: Dict[str, Path] = {}
    incomplete: Dict[str, Path] = {}
    for entry in sorted(custom_dir.iterdir()):
        if not entry.is_dir() or entry.name.startswith((".", "_")):
            continue
        name = entry.name
        if not _VALID_NAME_RE.match(name):
            continue
        py_path = entry / f"{name}.py"
        if py_path.is_file():
            complete[name] = entry
        else:
            incomplete[name] = entry
    return complete, incomplete


def missing_required_files(name: str, folder: Path) -> list[str]:
    """Return the required filenames absent from ``folder`` (empty when complete).

    The only strictly required file is ``<name>.py`` — ``<name>.prompty`` is
    optional and merely selects the LLM-judge kind. Used to build actionable
    "missing required file" messages for incomplete custom-evaluator folders.
    """
    missing: list[str] = []
    if not (folder / f"{name}.py").is_file():
        missing.append(f"{name}.py")
    return missing


def load_custom_evaluator(name: str, folder: Path) -> CustomEvaluatorSpec:
    """Import one referenced evaluator and resolve its top-level class.

    The lazy load phase (FR-004): executes the user's ``.py`` and runs the
    best-effort security scanner (FR-029). Call only for referenced evaluators.

    The kind is auto-detected from a sibling ``<name>.prompty``: present →
    LLM-judge (``is_llm=True``, prompty is scanned); absent → code-only
    (``is_llm=False``). For a code-only wrapper whose source looks LLM-shaped, a
    non-blocking warning is emitted in case the prompty was forgotten.

    Raises:
        CustomEvaluatorDiscoveryError: ``<name>.py`` is missing, or the module
            has a syntax error, fails to import, or defines no top-level class.
    """
    py_path = folder / f"{name}.py"
    prompty_path = folder / f"{name}.prompty"
    # Re-check the required .py: it existed at enumeration time, but may have
    # been removed before this lazy load.
    if not py_path.is_file():
        raise CustomEvaluatorDiscoveryError(
            f"Custom evaluator '{name}' is missing required file "
            f"'{py_path.name}'. Each custom evaluator needs a '{name}.py' "
            f"(and optionally a '{name}.prompty' for LLM-judge evaluators)."
        )

    is_llm = prompty_path.is_file()
    user_class = _import_user_class(name, py_path)

    # FR-029: best-effort security scanner; warnings are logged, never block.
    warnings = list(scan_python_file(py_path))
    if is_llm:
        warnings.extend(scan_prompty_file(prompty_path))
    for warning in warnings:
        emit_structured_log(
            "warning",
            f"[security:{warning.category}] {warning.file}:"
            f"{warning.line or '?'} — {warning.message}",
            operation=Operation.SETUP,
        )

    # Guard against a forgotten prompty: a code-only wrapper that references
    # LLM machinery is most likely a mis-saved LLM-judge evaluator.
    if not is_llm and _looks_like_llm_code(py_path):
        emit_structured_log(
            "warning",
            f"Custom evaluator '{name}' has no '{name}.prompty' and is treated "
            f"as a code-only (non-LLM) evaluator, but its code references LLM "
            f"machinery (load_flow/promptflow/model_config/.prompty). If this "
            f"is an LLM-judge evaluator, add '{name}.prompty' to its folder. "
            f"This is a best-effort heuristic and may be a false positive "
            f"(e.g. an unrelated string literal containing '.prompty'); if so, "
            f"this warning can be ignored.",
            operation=Operation.SETUP,
        )

    return CustomEvaluatorSpec(
        name=name,
        py_path=py_path,
        user_class=user_class,
        is_llm=is_llm,
        prompty_path=prompty_path if is_llm else None,
    )


# Identifier tells that a "code-only" wrapper is actually an LLM-judge whose
# prompty was forgotten (matched against real code references, not prose).
_LLM_CODE_TELL_NAMES = frozenset({"load_flow", "promptflow", "model_config"})


def _looks_like_llm_code(py_path: Path) -> bool:
    """Best-effort check whether a .py references LLM-judge machinery.

    Uses an AST walk so that mentions inside docstrings or comments (e.g. this
    very module, or a code-only example that *describes* LLM machinery in prose)
    do not trigger a false positive. Only real code references count:

    * identifiers / attributes / import names / keyword args matching
      ``load_flow`` / ``promptflow`` / ``model_config``, or
    * a non-docstring string literal containing ``.prompty`` (e.g. a path the
      wrapper loads).

    Returns ``False`` if the file can't be read or parsed (the security scanner
    handles genuinely broken files).
    """
    try:
        source = py_path.read_text(encoding="utf-8")
    except OSError:
        return False
    try:
        tree = ast.parse(source)
    except SyntaxError:
        return False

    # Collect the id() of docstring constant nodes so they're excluded below.
    docstring_ids = set()
    for node in ast.walk(tree):
        if isinstance(node, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
            body = getattr(node, "body", None)
            if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) and isinstance(body[0].value.value, str):
                docstring_ids.add(id(body[0].value))

    for node in ast.walk(tree):
        if isinstance(node, ast.Name) and node.id in _LLM_CODE_TELL_NAMES:
            return True
        if isinstance(node, ast.Attribute) and node.attr in _LLM_CODE_TELL_NAMES:
            return True
        if isinstance(node, ast.keyword) and node.arg in _LLM_CODE_TELL_NAMES:
            return True
        if isinstance(node, (ast.Import, ast.ImportFrom)):
            module = getattr(node, "module", "") or ""
            names = [module] + [a.name for a in node.names]
            if any("promptflow" in n for n in names):
                return True
        if (
            isinstance(node, ast.Constant)
            and isinstance(node.value, str)
            and id(node) not in docstring_ids
            and ".prompty" in node.value
        ):
            return True
    return False


# ── internal helpers ─────────────────────────────────────────────────


def _import_user_class(name: str, py_path: Path) -> Type:
    """Dynamically import a custom evaluator module and return its top-level class.

    The convention: the module exports a single top-level class that the CLI
    instantiates with keyword arguments (``model_config``, ``threshold``,
    ``options``). If multiple top-level classes exist, the first one
    matching the folder name (case-insensitive, with optional 'Evaluator'
    suffix) is preferred; otherwise the first class defined in the module
    is used.
    """
    module_name = f"_custom_evaluator_{name}"
    spec = importlib.util.spec_from_file_location(module_name, py_path)
    if spec is None or spec.loader is None:
        raise CustomEvaluatorDiscoveryError(
            f"Custom evaluator '{name}' module at {py_path} could not be loaded."
        )
    module = importlib.util.module_from_spec(spec)
    # Register before exec so the module can resolve its own qualified name —
    # required by dataclasses(slots=True), enums, pickling, and
    # typing.get_type_hints, which all look the module up in sys.modules.
    sys.modules[module_name] = module
    try:
        spec.loader.exec_module(module)
    except Exception as exc:  # noqa: BLE001 — surface import errors clearly
        sys.modules.pop(module_name, None)
        raise CustomEvaluatorDiscoveryError(
            f"Custom evaluator '{name}' failed to import: {exc}. "
            "Check the module for syntax errors and missing dependencies."
        ) from exc

    # Iterate in definition order (vars() preserves it) so the documented
    # "first class defined" fallback is honoured. inspect.getmembers sorts
    # alphabetically and would silently break that contract.
    classes = [
        obj
        for obj in vars(module).values()
        if inspect.isclass(obj) and obj.__module__ == module_name
    ]
    if not classes:
        raise CustomEvaluatorDiscoveryError(
            f"Custom evaluator '{name}' module at {py_path} does not define "
            "any top-level classes. Define a class with __init__(threshold, "
            "options) — plus model_config for LLM-judge evaluators — and a "
            "__call__(**kwargs) method."
        )

    expected = name.lower()
    preferred = [
        cls for cls in classes
        if cls.__name__.lower() in (expected, f"{expected}evaluator")
    ]
    return preferred[0] if preferred else classes[0]