"""Microsoft Foundry cloud (server-side) evaluation for LLM evaluators.

This module runs the LLM-based evaluators (Relevance, Coherence, Groundedness,
Similarity) through **Foundry cloud evaluation** using the Microsoft Foundry SDK
(``azure-ai-projects``) and the OpenAI Evals API, per
https://learn.microsoft.com/azure/foundry/how-to/develop/cloud-evaluation.

Why: the local Azure AI Evaluation SDK evaluators call Chat Completions with a
``response_format`` parameter that Responses-API models (gpt-5x / o-series)
reject (``400 unsupported_parameter``). Foundry cloud evaluation runs the same
built-in evaluators server-side, so it works with gpt-5x models — which is why
it is required for them.

Selection is presence-based (see ``common.should_use_foundry_eval``): whenever
both ``AZURE_AI_PROJECT_ENDPOINT`` and ``AZURE_AI_MODEL_NAME`` are set, the LLM
evaluators route through Foundry cloud evaluation regardless of the model. With no project
endpoint set, the LLM evaluators use the local evaluator path (gpt-4x only;
gpt-5x fails there on ``response_format``).

This is intentionally independent of ``judge_backend`` (the experimental GitHub
Copilot workaround): it is not a ``JudgeBackend`` and does not share its routing.

Flow (batch):
  1. Collect one row per (query, response, context, ground_truth) with the set of
     LLM evaluators requested for it (see :class:`FoundryEvalCollector`).
  2. Group rows by their evaluator set and, per group, create one evaluation
     (``evals.create``) + run (``evals.runs.create`` with inline ``file_content``),
     poll to completion, and read ``output_items``.
  3. Return ``{row_id: {metric_id: {"score", "reason"} | {"error"}}}`` so the
     pipeline can patch scores back and re-derive status.
"""

import threading
import time
from typing import Any, Dict, List, Optional, Tuple

from cli_logging.cli_logger import emit_structured_log
from cli_logging.logging_utils import Operation
from common import (
    METRIC_IDS,
    RELEVANCE,
    COHERENCE,
    GROUNDEDNESS,
    SIMILARITY,
)

# LLM evaluators routed through Foundry cloud evaluation, mapped to their Foundry
# built-in evaluator id and the data mapping (which item fields each needs).
# Data mappings mirror the LOCAL path semantics: Groundedness uses the expected
# response as `context`; Similarity uses it as `ground_truth`.
FOUNDRY_EVALUATORS: Dict[str, Dict[str, Any]] = {
    RELEVANCE: {
        "builtin": "builtin.relevance",
        "mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
    },
    COHERENCE: {
        "builtin": "builtin.coherence",
        "mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
    },
    GROUNDEDNESS: {
        "builtin": "builtin.groundedness",
        "mapping": {
            "query": "{{item.query}}",
            "response": "{{item.response}}",
            "context": "{{item.context}}",
        },
    },
    SIMILARITY: {
        "builtin": "builtin.similarity",
        "mapping": {
            "query": "{{item.query}}",
            "response": "{{item.response}}",
            "ground_truth": "{{item.ground_truth}}",
        },
    },
}

# Evaluator names this module can score (used by the runner to decide what to defer).
FOUNDRY_LLM_EVALUATORS = frozenset(FOUNDRY_EVALUATORS.keys())

# Polling configuration for a Foundry evaluation run.
_POLL_INITIAL_DELAY_SECONDS = 3.0
_POLL_MAX_DELAY_SECONDS = 20.0
_POLL_BACKOFF_FACTOR = 1.5
_POLL_MAX_WAIT_SECONDS = 1800.0
_TERMINAL_STATUSES = frozenset({"completed", "failed", "canceled", "cancelled", "error"})


class FoundryEvalCollector:
    """Thread-safe collector of rows to evaluate via Foundry cloud evaluation.

    Rows are coalesced by ``(query, response, context, ground_truth)`` so a single
    item requesting several evaluators produces one row carrying the union of its
    evaluators. Identical content across items also dedups to one row.
    """

    def __init__(self) -> None:
        self._lock = threading.Lock()
        self._by_key: Dict[Tuple[str, str, str, str], Dict[str, Any]] = {}
        self._rows: List[Dict[str, Any]] = []

    def register(
        self,
        *,
        query: str,
        response: str,
        context: str,
        ground_truth: str,
        metric: str,
    ) -> int:
        """Register a metric request for a row; returns the row's stable id."""
        key = (query or "", response or "", context or "", ground_truth or "")
        with self._lock:
            row = self._by_key.get(key)
            if row is None:
                row = {
                    "row_id": len(self._rows),
                    "query": key[0],
                    "response": key[1],
                    "context": key[2],
                    "ground_truth": key[3],
                    "metrics": set(),
                }
                self._by_key[key] = row
                self._rows.append(row)
            row["metrics"].add(metric)
            return row["row_id"]

    @property
    def rows(self) -> List[Dict[str, Any]]:
        return self._rows

    def is_empty(self) -> bool:
        return not self._rows


class FoundryCloudEvaluator:
    """Runs LLM evaluators through Microsoft Foundry cloud evaluation.

    Args:
        project_endpoint: Foundry project endpoint
            (``https://<account>.services.ai.azure.com/api/projects/<project>``).
        deployment: The Azure OpenAI model deployment name used by the AI-assisted
            evaluators (for example ``gpt-5-mini``).
        log_level: CLI log level ("info" | "debug" | ...).
    """

    def __init__(self, project_endpoint: str, deployment: str, log_level: str = "info") -> None:
        self._project_endpoint = project_endpoint
        self._deployment = deployment
        self._log_level = log_level
        self._lock = threading.Lock()
        self._openai_client: Optional[Any] = None
        self._project_client: Optional[Any] = None

    def describe(self) -> str:
        """Evaluator description for the run's output metadata."""
        return f"Microsoft Foundry cloud evaluation (deployment: {self._deployment})"

    def _get_client(self) -> Any:
        """Lazily construct and cache the Foundry OpenAI evals client (thread-safe)."""
        if self._openai_client is not None:
            return self._openai_client
        with self._lock:
            if self._openai_client is None:
                from azure.identity import DefaultAzureCredential
                from azure.ai.projects import AIProjectClient

                self._project_client = AIProjectClient(
                    endpoint=self._project_endpoint,
                    credential=DefaultAzureCredential(),
                )
                self._openai_client = self._project_client.get_openai_client()
        return self._openai_client

    def verify(self) -> None:
        """Construct the Foundry client up front to catch construction errors."""
        self._get_client()

    def evaluate_batch(self, rows: List[Dict[str, Any]]) -> Dict[int, Dict[str, Dict[str, Any]]]:
        """Evaluate all collected rows, grouped by evaluator set.

        Returns ``{row_id: {metric_id: {"score", "reason"} | {"error"}}}``.
        """
        results: Dict[int, Dict[str, Dict[str, Any]]] = {}
        if not rows:
            return results

        # Group rows by their (frozen) evaluator set so each run applies exactly
        # the evaluators the rows asked for (avoids missing-field errors and cost).
        groups: Dict[frozenset, List[Dict[str, Any]]] = {}
        for row in rows:
            key = frozenset(row["metrics"])
            groups.setdefault(key, []).append(row)

        emit_structured_log(
            "info",
            f"☁️  Foundry cloud evaluation: {len(rows)} row(s) across "
            f"{len(groups)} run(s) (deployment={self._deployment}).",
            operation=Operation.EVALUATE,
        )

        for metric_set, group_rows in groups.items():
            self._evaluate_group(sorted(metric_set), group_rows, results)
        return results

    def _evaluate_group(
        self,
        metrics: List[str],
        group_rows: List[Dict[str, Any]],
        results: Dict[int, Dict[str, Dict[str, Any]]],
    ) -> None:
        """Run one Foundry evaluation for a set of rows sharing the same metrics."""
        metric_ids = [METRIC_IDS[m] for m in metrics]
        try:
            client = self._get_client()
            testing_criteria = [
                {
                    "type": "azure_ai_evaluator",
                    "name": METRIC_IDS[m],
                    "evaluator_name": FOUNDRY_EVALUATORS[m]["builtin"],
                    "initialization_parameters": {"model": self._deployment},
                    "data_mapping": FOUNDRY_EVALUATORS[m]["mapping"],
                }
                for m in metrics
            ]
            data_source_config = {
                "type": "custom",
                "item_schema": {
                    "type": "object",
                    "properties": {
                        "query": {"type": "string"},
                        "response": {"type": "string"},
                        "context": {"type": "string"},
                        "ground_truth": {"type": "string"},
                    },
                    "required": ["query", "response"],
                },
            }
            content = [
                {
                    "item": {
                        "query": row["query"],
                        "response": row["response"],
                        "context": row["context"],
                        "ground_truth": row["ground_truth"],
                    }
                }
                for row in group_rows
            ]
            # datasource_item_id in the results corresponds to the 0-based index
            # of each item in `content`; map it back to our row_id.
            index_to_row_id = {i: row["row_id"] for i, row in enumerate(group_rows)}

            eval_object = client.evals.create(
                name="runevals-foundry",
                data_source_config=data_source_config,
                testing_criteria=testing_criteria,
            )
            run = client.evals.runs.create(
                eval_id=eval_object.id,
                name="runevals-foundry-run",
                data_source={
                    "type": "jsonl",
                    "source": {"type": "file_content", "content": content},
                },
            )
            final = self._wait_for_conclusion(client, eval_object.id, run.id)
            if getattr(final, "status", None) != "completed":
                detail = getattr(getattr(final, "error", None), "message", None) or getattr(
                    final, "status", "unknown"
                )
                self._mark_group_error(
                    group_rows, metric_ids, f"Foundry evaluation run did not complete: {detail}", results
                )
                return

            self._collect_group_results(client, eval_object.id, run.id, index_to_row_id, results)
        except Exception as e:  # noqa: BLE001 — surfaced as per-row errored scores
            emit_structured_log(
                "error",
                f"Foundry cloud evaluation failed for metrics {metric_ids}: {e}",
                operation=Operation.EVALUATE,
            )
            self._mark_group_error(group_rows, metric_ids, f"Foundry evaluation failed: {e}", results)

    def _collect_group_results(
        self,
        client: Any,
        eval_id: str,
        run_id: str,
        index_to_row_id: Dict[int, int],
        results: Dict[int, Dict[str, Dict[str, Any]]],
    ) -> None:
        """Read output items for a completed run and record per-row metric scores."""
        unmapped = 0
        for item in client.evals.runs.output_items.list(eval_id=eval_id, run_id=run_id):
            ds_index = _coerce_index(getattr(item, "datasource_item_id", None))
            row_id = index_to_row_id.get(ds_index) if ds_index is not None else None
            if row_id is None:
                unmapped += 1
                continue
            row_scores = results.setdefault(row_id, {})
            for result in getattr(item, "results", []) or []:
                rd = result if isinstance(result, dict) else _to_dict(result)
                metric_id = rd.get("metric") or rd.get("name")
                if not metric_id:
                    continue
                metric_id = str(metric_id).lower()
                score = rd.get("score")
                reason = rd.get("reason", "")
                if score is None:
                    row_scores[metric_id] = {"error": rd.get("reason") or "Foundry returned no score."}
                else:
                    try:
                        row_scores[metric_id] = {"score": float(score), "reason": reason}
                    except (TypeError, ValueError):
                        row_scores[metric_id] = {"error": f"Non-numeric Foundry score: {score!r}"}
        if unmapped:
            emit_structured_log(
                "warning",
                f"Foundry cloud evaluation: {unmapped} output item(s) could not "
                f"be mapped to a row (unexpected datasource_item_id); their "
                f"scores were dropped.",
                operation=Operation.EVALUATE,
            )

    @staticmethod
    def _mark_group_error(
        group_rows: List[Dict[str, Any]],
        metric_ids: List[str],
        message: str,
        results: Dict[int, Dict[str, Dict[str, Any]]],
    ) -> None:
        for row in group_rows:
            row_scores = results.setdefault(row["row_id"], {})
            for metric_id in metric_ids:
                row_scores.setdefault(metric_id, {"error": message})

    def _wait_for_conclusion(self, client: Any, eval_id: str, run_id: str) -> Any:
        """Poll a run with exponential backoff until it reaches a terminal state."""
        waited = 0.0
        delay = _POLL_INITIAL_DELAY_SECONDS
        while waited < _POLL_MAX_WAIT_SECONDS:
            run = client.evals.runs.retrieve(run_id=run_id, eval_id=eval_id)
            if getattr(run, "status", None) in _TERMINAL_STATUSES:
                return run
            time.sleep(delay)
            waited += delay
            delay = min(delay * _POLL_BACKOFF_FACTOR, _POLL_MAX_DELAY_SECONDS)
        raise TimeoutError(
            f"Foundry evaluation run {eval_id}/{run_id} did not conclude within "
            f"{int(_POLL_MAX_WAIT_SECONDS)}s."
        )


def _coerce_index(value: Any) -> Optional[int]:
    """Coerce a Foundry ``datasource_item_id`` to a 0-based int index.

    The results API returns the 0-based position of each item in the submitted
    ``content`` list. Accept ints and integer-valued strings; anything else
    (e.g. an opaque id from a future API change) returns ``None`` so the caller
    can flag it as unmapped rather than silently dropping scores.
    """
    if isinstance(value, bool):
        return None
    if isinstance(value, int):
        return value
    if isinstance(value, str):
        try:
            return int(value.strip())
        except (TypeError, ValueError):
            return None
    return None


def _to_dict(obj: Any) -> Dict[str, Any]:
    """Best-effort conversion of an SDK result object to a plain dict."""
    if hasattr(obj, "model_dump"):
        return obj.model_dump()
    if hasattr(obj, "dict"):
        return obj.dict()
    if hasattr(obj, "__dict__"):
        return vars(obj)
    return {}