# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Reporting logic for CXAS skill benchmarking."""
import dataclasses
import datetime
import json
import os
import pathlib
import re
import typing
import urllib.parse
from collections.abc import Sequence
from typing import Any
import markdown
from skill_eval import benchmark, scenario
_TRAJECTORY_URL_TEMPLATE = os.environ.get("TRAJECTORY_VIEWER_URL", "")
_INDEX_HTML_TEMPLATE = """
Agent Evaluation Benchmark
{history}
{log_links}
{latency_table}
"""
class _ReportEncoder(json.JSONEncoder):
"""Custom JSON encoder for benchmark results."""
def default(self, o: Any) -> Any:
if isinstance(o, benchmark.ExecutionStatus):
return o.value
if isinstance(o, datetime.timedelta):
return o.total_seconds()
return super().default(o)
def generate_json_report(result: benchmark.ConversationResult) -> str:
"""Generates a serialized JSON representation of the result."""
return json.dumps(dataclasses.asdict(result), indent=2, cls=_ReportEncoder)
def _escape_html(text: Any) -> str:
"""Escapes text for safe HTML rendering."""
if text is None:
return ""
s = str(text)
return (
s.replace("&", "&")
.replace("<", "<")
.replace(">", ">")
.replace('"', """)
.replace("'", "'")
)
def _format_ts(ts: float) -> str:
"""Formats a unix timestamp for display."""
if ts <= 0:
return "N/A"
return datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
def generate_index_html(
results: Sequence[benchmark.ConversationResult],
suite_duration: datetime.timedelta = datetime.timedelta(),
) -> str:
"""Generates the main index.html summary page."""
by_scenario = _group_by_scenario(results)
all_heads = sorted(list(set(r.head_name for r in results)))
eval_start_val = min(
(r.start_time for r in results if r.start_time > 0), default=0
)
eval_end_val = max(
(r.end_time for r in results if r.end_time > 0), default=0
)
eval_start = _format_ts(eval_start_val)
eval_end = _format_ts(eval_end_val)
head_headers = "".join(f"
{_escape_html(h)}
" for h in all_heads)
rows_html = ""
if not results:
rows_html = (
'
Initializing benchmark suite...
'
)
for scenario_name in sorted(by_scenario.keys()):
rows_html += (
f'
{_escape_html(scenario_name)}
'
)
head_map = {r.head_name: r for r in by_scenario[scenario_name]}
for head_name in all_heads:
res = head_map.get(head_name)
if not res:
rows_html += "
"
)
rubric_html = ""
if result.rubric_results:
res = result.rubric_results
earned, total = res.total_score, res.max_score
pct = (earned / total) if total > 0 else 0
if pct >= 1.0:
s_cls, s_txt = "status-pass", "PASSED"
elif pct >= 0.5:
s_cls, s_txt = "status-warning", "WARNING"
else:
s_cls, s_txt = "status-fail", "FAILED"
rows = ""
for s in res.scores:
score_style = (
"color: #137333; font-weight: bold;"
if s.score == 2
else ("color: #b05e00;" if s.score == 1 else "color: #c5221f;")
)
rows += (
'
{_escape_html(s.criteria)}
'
)
rows += (
'
{s.score}/2
'
)
rows += (
'
{_escape_html(s.reasoning)}
'
)
rubric_html = f"""
Rubric Assessment
{s_txt} ({earned}/{total})
Overall: {_escape_html(res.summary)}
Criterion
Score
Reasoning
{rows}
"""
history_html = ""
num_turns = (len(result.messages) + 1) // 2
for i in range(num_turns):
m = result.turn_metrics[i] if i < len(result.turn_metrics) else None
timing_html = ""
if m:
timing_html = (
'
'
)
elif (
i < num_turns - 1
or result.status == benchmark.ExecutionStatus.FAILED
):
history_html += (
'
Agent: [FAILED]
'
)
history_html += ""
final_failure = result.failure_reason
if (
not final_failure
and result.messages
and "[[FAIL" in result.messages[-1]["content"]
):
final_failure = result.messages[-1]["content"]
if final_failure:
history_html += (
'