""" CitationsEvaluator - A custom evaluator for analyzing citations in M365 Copilot responses. This evaluator uses regex-based pattern matching to detect citations in four modes: 1. OAI_UNICODE: New OAI format: \ue200cite\ue202turn{X}search{Y}\ue201 2. LEGACY_BRACKET: Old format: [^i^] where i is the citation index 3. MARKDOWN_LINK: Markdown link format: [N](targetLink#suffix) numbered anchor links 4. AUTO: Automatically detects all formats simultaneously Where X, Y, i, and N are natural numbers representing conversation turn, search result index, or citation index, and suffix is the reference id that keys the response's structured references. """ import re from enum import Enum from typing import Dict, Any, Optional class CitationFormat(Enum): """Enum for different citation formats supported by the evaluator.""" OAI_UNICODE = "oai_unicode" # New format: \ue200cite\ue202turn{X}search{Y}\ue201 LEGACY_BRACKET = "bracket" # Old format: [^i^] MARKDOWN_LINK = "markdown" # Markdown link format: [N](targetLink#suffix) AUTO = "mixed" # Automatically detect all formats class CitationsEvaluator: """ A custom evaluator that analyzes citations in response text without using an LLM. This evaluator detects citation patterns and returns: - Whether at least one citation is present - The number of unique citations found Supports four modes: - OAI_UNICODE: Detects only OAI unicode format citations - LEGACY_BRACKET: Detects only legacy bracket format citations - MARKDOWN_LINK: Detects only markdown link format citations ([N](targetLink#suffix)) - AUTO: Automatically detects all formats simultaneously """ def __init__(self, citation_format: CitationFormat = CitationFormat.OAI_UNICODE): """ Initialize the CitationsEvaluator with the specified citation format. Args: citation_format (CitationFormat): The format of citations to detect. Defaults to OAI_UNICODE format. """ self.citation_format = citation_format oai_pattern = r'\ue200cite\ue202turn\d+search\d+\ue201' legacy_pattern = r'\[\^\d+\^\]' # Markdown link: [N](targetLink#suffix) — numeric label + '#suffix' fragment required. markdown_pattern = r'\[\d+\]\(https?://[^)\s]+#[^)\s#]+\)' if citation_format == CitationFormat.OAI_UNICODE: # Pattern to match citations: \ue200cite\ue202turn{number}search{number}\ue201 self.citation_pattern = oai_pattern elif citation_format == CitationFormat.LEGACY_BRACKET: # Pattern to match citations: [^number^] self.citation_pattern = legacy_pattern elif citation_format == CitationFormat.MARKDOWN_LINK: # Pattern to match citations: [number](targetLink#suffix) self.citation_pattern = markdown_pattern elif citation_format == CitationFormat.AUTO: # Auto-detect all formats using alternation (|) self.citation_pattern = rf'(?:{oai_pattern})|(?:{legacy_pattern})|(?:{markdown_pattern})' else: raise ValueError(f"Unsupported citation format: {citation_format}") # Compile the pattern once after determining which format to use self.compiled_pattern = re.compile(self.citation_pattern) def __call__(self, *, response: str, **kwargs) -> Dict[str, Any]: """ Evaluate the response text for citations. Args: response (str): The response text from the M365 Copilot agent **kwargs: Additional keyword arguments (not used but kept for compatibility) Returns: Dict[str, Any]: Evaluation results containing: - citation_format (str): The format used for detection - score (int): Number of unique citations found - result (str): "pass" if citations found, "fail" otherwise - threshold (int): Minimum threshold for passing (1) - reason (str): Explanation of the result with citation details """ if not isinstance(response, str): response = str(response) if response is not None else "" # Find all citations and get unique ones (same for all modes). # dict.fromkeys preserves first-occurrence order, so the displayed # citation order/labels are deterministic and match the response. citation_matches = self.compiled_pattern.findall(response) unique_citations = list(dict.fromkeys(citation_matches)) # Initialize citation details list (used by all modes) citation_details = [] # Initialize counters only for AUTO mode if self.citation_format == CitationFormat.AUTO: oai_count = 0 legacy_count = 0 markdown_count = 0 # Markdown citations are deduped by their '#suffix' reference id, so the # same source cited with different numeric labels counts once. seen_markdown_suffixes = set() # Process all citations (unified extraction logic) for citation in unique_citations: # Determine citation type and extract details if '\ue200' in citation: # OAI format (contains start marker) turn_search_match = re.search(r'turn(\d+)search(\d+)', citation) if turn_search_match: turn_num = turn_search_match.group(1) search_num = turn_search_match.group(2) # Add appropriate prefix based on mode if self.citation_format == CitationFormat.AUTO: citation_details.append(f"oai:turn{turn_num}search{search_num}") oai_count += 1 else: # OAI_UNICODE mode citation_details.append(f"turn{turn_num}search{search_num}") elif '](' in citation: # Markdown link format; dedupe by the '#suffix' reference id markdown_match = re.search(r'#([^)\s#]+)\)', citation) if markdown_match: suffix = markdown_match.group(1) if suffix in seen_markdown_suffixes: continue seen_markdown_suffixes.add(suffix) # Display the numeric [N] label shown in the response rather # than the internal '#suffix' reference id. label_match = re.search(r'\[(\d+)\]', citation) label = f"[{label_match.group(1)}]" if label_match else suffix # Add appropriate prefix based on mode if self.citation_format == CitationFormat.AUTO: citation_details.append(f"markdown:{label}") markdown_count += 1 else: # MARKDOWN_LINK mode citation_details.append(label) else: # Legacy bracket format bracket_match = re.search(r'\[\^(\d+)\^\]', citation) if bracket_match: citation_num = bracket_match.group(1) # Add appropriate prefix based on mode if self.citation_format == CitationFormat.AUTO: citation_details.append(f"legacy:citation{citation_num}") legacy_count += 1 else: # LEGACY_BRACKET mode citation_details.append(f"citation{citation_num}") format_info = None if self.citation_format == CitationFormat.AUTO: format_info = f"OAI: {oai_count}, Legacy: {legacy_count}, Markdown: {markdown_count}" # Build results (markdown suffix-dedupe means we count details, not raw matches) total_citations = len(citation_details) # Construct reason string with optional format info reason_parts = [f"Found {total_citations} unique citation(s)"] if format_info: reason_parts.append(f"[{format_info}]:") else: reason_parts.append(":") reason_parts.append(', '.join(citation_details) if citation_details else 'None') results = { "citation_format": self.citation_format.value, "citations": total_citations, "result": "pass" if total_citations > 0 else "fail", "threshold": 1, "reason": " ".join(reason_parts) } return results def get_name(self) -> str: """Return the name of this evaluator.""" return "CitationsEvaluator" def get_description(self) -> str: """Return a description of what this evaluator does.""" return f"Analyzes response text for M365 Copilot citations using regex pattern matching ({self.citation_format.value} format)" def citations_evaluator(*, response: str, citation_format: CitationFormat = CitationFormat.OAI_UNICODE, **kwargs) -> Dict[str, Any]: """ Standalone function wrapper for the CitationsEvaluator. This function provides a simple interface compatible with Azure AI Evaluation SDK. Args: response (str): The response text to evaluate citation_format (CitationFormat): The format of citations to detect **kwargs: Additional keyword arguments Returns: Dict[str, Any]: Citation evaluation results """ evaluator = CitationsEvaluator(citation_format=citation_format) return evaluator(response=response, **kwargs) # For convenience, export the main classes and functions __all__ = ['CitationsEvaluator', 'CitationFormat', 'citations_evaluator']