"""Evaluator resolution module for per-prompt evaluator configuration.

Resolves which evaluators to run on each prompt by merging prompt-level config
with file-level defaults and system defaults, following extend/replace modes.

Also exposes the runtime registry that combines built-in evaluators with
user-authored custom evaluators discovered from
``<project_root>/custom-evaluators/`` (see ``custom_evaluators/discovery.py``).
"""

import difflib
from pathlib import Path
from typing import Any, Dict, Optional

from cli_logging.cli_logger import emit_structured_log
from cli_logging.logging_utils import Operation
from common import (
    RELEVANCE,
    COHERENCE,
    GROUNDEDNESS,
    SIMILARITY,
    CITATIONS,
    EXACT_MATCH,
    PARTIAL_MATCH,
    RETRIEVAL_QUERY,
    RETRIEVAL_RESULT,
    BUILTIN_EVALUATOR_NAMES,
    SYSTEM_DEFAULT_EVALUATORS,
    RegistryEntry,
)
from custom_evaluators.discovery import (
    CustomEvaluatorDiscoveryError,
    CustomEvaluatorSpec,
    enumerate_custom_evaluator_folders,
    load_custom_evaluator,
    missing_required_files,
)


# Static registry of available evaluators per data-model.md
EVALUATOR_REGISTRY: Dict[str, RegistryEntry] = {
    RELEVANCE: RegistryEntry(type="llm", default_threshold=3),
    COHERENCE: RegistryEntry(type="llm", default_threshold=3),
    GROUNDEDNESS: RegistryEntry(type="llm", default_threshold=3),
    SIMILARITY: RegistryEntry(type="llm", default_threshold=3),
    CITATIONS: RegistryEntry(type="non-llm", default_threshold=1),
    EXACT_MATCH: RegistryEntry(type="non-llm", default_threshold=None),
    PARTIAL_MATCH: RegistryEntry(type="non-llm", default_threshold=0.5),
    RETRIEVAL_QUERY: RegistryEntry(type="non-llm", default_threshold=1.0),
    RETRIEVAL_RESULT: RegistryEntry(type="non-llm", default_threshold=1.0),
}

# Evaluators whose threshold is pinned by contract and cannot be overridden.
_FIXED_THRESHOLD_EVALUATORS = frozenset({RETRIEVAL_QUERY, RETRIEVAL_RESULT})

# Built-in evaluator names, re-exported from common so the canonical list lives
# in one place. Used for case-insensitive collision detection (FR-030) and
# built-in-vs-custom reporting. A unit test asserts every EVALUATOR_REGISTRY
# name is in this set.
_BUILTIN_EVALUATOR_NAMES = BUILTIN_EVALUATOR_NAMES

# Structurally-complete custom-evaluator folders (name → path), filled by the
# cheap startup :func:`enumerate_custom_evaluators` pass. Not yet imported;
# import happens lazily in :func:`get_custom_evaluator_spec`.
CUSTOM_EVALUATOR_FOLDERS: Dict[str, Path] = {}

# Import cache of loaded custom-evaluator specs (name → spec) for
# :func:`get_custom_evaluator_spec`. Empty at startup; populated on first load.
CUSTOM_EVALUATOR_REGISTRY: Dict[str, CustomEvaluatorSpec] = {}

# Problem folders found at startup (name → ready-to-raise reason), for the two
# issues detectable without importing user code: a built-in collision (FR-030)
# or a missing required file. Keyed by the exact folder name; the reference-time
# lookup in :func:`validate_evaluator_names` is case-insensitive (FR-030). Not
# registered; enforced lazily — a referenced one is rejected, an unreferenced
# one skipped. Syntax/import errors of complete folders surface later as inline
# errors (FR-026).
FAILED_CUSTOM_EVALUATORS: Dict[str, str] = {}


def enumerate_custom_evaluators(
    root_dir: Optional[Path] = None,
) -> Dict[str, Path]:
    """Cheaply enumerate user-authored custom evaluators (FR-001, FR-004, FR-030).

    Scans ``<root_dir>/custom-evaluators/`` (default: cwd) WITHOUT importing
    user code or running the scanner — import + scan happen lazily, per
    referenced evaluator, in :func:`get_custom_evaluator_spec` (FR-004).

    **Non-fatal** (FR-024): never raises. Complete, non-colliding folders are
    registered in :data:`CUSTOM_EVALUATOR_FOLDERS` and :data:`EVALUATOR_REGISTRY`.
    Problem folders — built-in collisions (FR-030) or those missing a required
    file — are recorded in :data:`FAILED_CUSTOM_EVALUATORS` with a ready-to-raise
    reason and enforced lazily: rejected by :func:`validate_evaluator_names` only
    if referenced, silently skipped otherwise.

    Call once at CLI startup. Returns the registered name → folder map (for the
    debug "discovered" listing).
    """
    complete, incomplete = enumerate_custom_evaluator_folders(root_dir)
    builtin_lower = {b.lower(): b for b in BUILTIN_EVALUATOR_NAMES}

    for name, folder in complete.items():
        builtin = builtin_lower.get(name.lower())
        if builtin is not None:
            FAILED_CUSTOM_EVALUATORS[name] = _collision_message(name, builtin)
            emit_structured_log(
                "debug",
                f"Skipped custom evaluator folder '{name}' (collides with "
                f"built-in '{builtin}'); will error only if referenced.",
                operation=Operation.SETUP,
            )
            continue
        CUSTOM_EVALUATOR_FOLDERS[name] = folder
        # Kind is auto-detected from the presence of a sibling <name>.prompty:
        # present → LLM-judge (1-5 scale, default threshold 3); absent →
        # code-only / non-LLM (any-numeric score, default threshold 1).
        is_llm = (folder / f"{name}.prompty").is_file()
        EVALUATOR_REGISTRY[name] = RegistryEntry(
            type="custom-llm" if is_llm else "custom-non-llm",
            default_threshold=3 if is_llm else 1,
        )

    for name, folder in incomplete.items():
        builtin = builtin_lower.get(name.lower())
        if builtin is not None:
            # Collision takes priority over incompleteness.
            FAILED_CUSTOM_EVALUATORS[name] = _collision_message(name, builtin)
            emit_structured_log(
                "debug",
                f"Skipped incomplete custom evaluator folder '{name}' "
                f"(collides with built-in '{builtin}').",
                operation=Operation.SETUP,
            )
            continue
        FAILED_CUSTOM_EVALUATORS[name] = _incomplete_message(name, folder)
        emit_structured_log(
            "debug",
            f"Recorded incomplete custom evaluator folder '{name}'; will "
            f"error only if referenced.",
            operation=Operation.SETUP,
        )

    return dict(CUSTOM_EVALUATOR_FOLDERS)


def _collision_message(name: str, builtin: str) -> str:
    """Build the reference-time error for a folder that shadows a built-in (FR-030)."""
    return (
        f"Custom evaluator name '{name}' collides with built-in evaluator "
        f"'{builtin}' (collision detection is case-insensitive). Rename the "
        f"folder — for example, prefix it with your organization or domain, "
        f"e.g. 'my_{name.lower()}'."
    )


def _incomplete_message(name: str, folder: Path) -> str:
    """Build the reference-time error for a folder missing a required file."""
    missing = missing_required_files(name, folder)
    return (
        f"Custom evaluator '{name}' is incomplete — missing required "
        f"file(s): {', '.join(missing)}. Each custom evaluator needs a "
        f"'{name}.py' in custom-evaluators/{name}/ (add a '{name}.prompty' "
        f"only for LLM-judge evaluators)."
    )


def get_custom_evaluator_spec(name: str) -> CustomEvaluatorSpec:
    """Lazily import, scan, and cache the spec for one custom evaluator.

    On first call: imports the ``.py``, resolves its class, runs the security
    scanner (FR-029), and caches in :data:`CUSTOM_EVALUATOR_REGISTRY`. Later
    calls return the cache (warnings emitted at most once).

    Raises:
        CustomEvaluatorDiscoveryError: never enumerated, or fails to import
            (missing file, syntax error, missing dependency, no class). The
            runner turns this into an inline ``error`` result (FR-026).
    """
    cached = CUSTOM_EVALUATOR_REGISTRY.get(name)
    if cached is not None:
        return cached

    folder = CUSTOM_EVALUATOR_FOLDERS.get(name)
    if folder is None:
        # Should not happen — validate_evaluator_names gates references to
        # known custom-evaluator names before invocation.
        raise CustomEvaluatorDiscoveryError(
            f"Custom evaluator '{name}' was not enumerated at startup."
        )

    spec = load_custom_evaluator(name, folder)
    CUSTOM_EVALUATOR_REGISTRY[name] = spec
    emit_structured_log(
        "debug",
        f"Loaded custom evaluator '{name}' (class={spec.user_class.__name__}).",
        operation=Operation.EVALUATE,
    )
    return spec


def validate_evaluator_names(evaluator_map: Dict[str, Any]) -> None:
    """Pre-flight check that every referenced evaluator name is usable (FR-030, FR-026).

    Run before any agent request (see ``evaluation_runner.run_pipeline``), so a
    problem rejects the whole run upfront. Per referenced name, in order:

    1. **Structural failure**: name matches — case-insensitively (FR-030) — a
       folder recorded in :data:`FAILED_CUSTOM_EVALUATORS` (a built-in collision
       or a missing required file). Checked first over *every* name, even ones
       resolving to a built-in, so a folder shadowing a built-in can't slip
       through when referenced in the built-in's own casing. Rejected with the
       stored reason (a rename suggestion or the missing-file list).
    2. **Known** (built-in, non-LLM, or complete custom): accepted.
    3. **Unknown**: rejected with a "valid evaluators" listing + "Did you mean?".

    Import failures (syntax error, missing dependency, no class) are NOT caught
    here — the name is registered, so they surface as inline ``error`` results
    at invocation (FR-026) and the run continues.
    """
    # 1. Structural problems found at startup (collision or missing file).
    # Matched case-insensitively (FR-030) and checked before the registry
    # early-return so a folder that shadows a built-in (its name IS in
    # EVALUATOR_REGISTRY as the built-in) is still rejected — even when
    # referenced in the built-in's own casing — instead of silently resolving
    # to the built-in. The stored reason distinguishes collision from missing file.
    failed_by_lower = {n.lower(): reason for n, reason in FAILED_CUSTOM_EVALUATORS.items()}
    for name in evaluator_map:
        reason = failed_by_lower.get(name.lower())
        if reason is not None:
            raise ValueError(reason)

    invalid_names = [name for name in evaluator_map if name not in EVALUATOR_REGISTRY]
    if not invalid_names:
        return

    # Categorize valid evaluators for the error message
    llm_evals = [n for n, r in EVALUATOR_REGISTRY.items() if r.type == "llm"]
    tool_evals = [n for n, r in EVALUATOR_REGISTRY.items() if r.type == "tool"]
    non_llm_evals = [n for n, r in EVALUATOR_REGISTRY.items() if r.type == "non-llm"]
    custom_evals = [n for n, r in EVALUATOR_REGISTRY.items() if r.type == "custom-llm"]
    custom_non_llm_evals = [n for n, r in EVALUATOR_REGISTRY.items() if r.type == "custom-non-llm"]

    lines = []
    for name in invalid_names:
        lines.append(f'Unknown evaluator "{name}".')
        close = difflib.get_close_matches(name, EVALUATOR_REGISTRY.keys(), n=1, cutoff=0.5)
        if close:
            lines.append(f'Did you mean "{close[0]}"?')

    lines.append("")
    lines.append("Valid evaluators are:")
    for category, label in [
        (llm_evals, "LLM-based"),
        (tool_evals, "tool evaluation"),
        (non_llm_evals, "non-LLM"),
        (custom_evals, "custom LLM"),
        (custom_non_llm_evals, "custom non-LLM"),
    ]:
        if category:
            lines.append(f"  - {', '.join(category)} ({label})")

    if not custom_evals and not custom_non_llm_evals:
        lines.append("")
        lines.append(
            "Tip: custom evaluators are discovered from "
            "<project_root>/custom-evaluators/<name>/. Make sure you're "
            "running the CLI from the project root."
        )

    raise ValueError("\n".join(lines))


def validate_evaluator_options(evaluator_map: Dict[str, Any]) -> None:
    """Reject per-evaluator option keys that aren't supported by the evaluator.
    """
    violations = []
    for eval_name, options in evaluator_map.items():
        if not isinstance(options, dict):
            continue
        if eval_name in _FIXED_THRESHOLD_EVALUATORS and "threshold" in options:
            pinned = EVALUATOR_REGISTRY[eval_name].default_threshold
            violations.append(
                f"'threshold' is not configurable on {eval_name} "
                f"(pinned at {pinned} by contract). Remove the key."
            )
    if violations:
        raise ValueError("\n".join(violations))


def resolve_default_evaluators(file_defaults: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
    """Resolve effective default evaluators, falling back to system defaults.

    Precedence: file-level defaults > system defaults.
    An explicit empty dict means "no default evaluators".
    """
    # File-level defaults (including explicit empty dict)
    if file_defaults is not None:
        return file_defaults

    # System defaults
    return {name: {} for name in SYSTEM_DEFAULT_EVALUATORS}


def resolve_evaluators_for_prompt(
    prompt_evaluators: Optional[Dict[str, Any]],
    evaluators_mode: str,
    prompt: str,
    default_evaluators: Dict[str, Any],
) -> Dict[str, Any]:
    """Resolve which evaluators to run for a single prompt.

    Args:
        prompt_evaluators: Per-prompt evaluator config (None if not specified).
        evaluators_mode: How to combine with defaults ("extend" or "replace").
        prompt: The prompt text (used in warning messages).
        default_evaluators: Resolved default evaluators (from resolve_default_evaluators).

    Returns:
        Resolved EvaluatorMap (dict of evaluator_name -> options).
    """
    # No prompt-level config → use defaults
    if prompt_evaluators is None:
        return dict(default_evaluators)

    if evaluators_mode == "replace":
        if not prompt_evaluators:
            emit_structured_log(
                "warning",
                f"Empty evaluators with 'replace' mode for prompt: "
                f"'{prompt[:80]}'. No evaluators will run.",
                operation=Operation.EVALUATE,
            )
        return dict(prompt_evaluators)

    # mode == "extend": merge defaults with prompt overrides (prompt wins on conflict)
    merged = dict(default_evaluators)
    merged.update(prompt_evaluators)
    return merged


def get_evaluator_threshold(evaluator_name: str, options: Dict[str, Any]) -> Optional[float]:
    """Get the threshold for an evaluator, with option override support."""
    if "threshold" in options:
        return options["threshold"]
    entry = EVALUATOR_REGISTRY.get(evaluator_name)
    return entry.default_threshold if entry else None