# ruff: noqa: PLR0917
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import importlib.util
import json
import os
import sys
import typing
from unittest.mock import ANY, mock_open, patch

import pandas as pd
import pytest

from cxas_scrapi.utils.eval_utils import (
    COMBINED_REPORT_FILENAME,
    COMBINED_REPORT_JSON_FILENAME,
    add_timestamp_suffix,
)
from cxas_scrapi.utils.latency_parser import LatencyParser
from cxas_scrapi.utils.reporting import (
    _escape,
    _fmt_duration,
    _format_trace_line,
    _load_sim_test_cases,
    _resolve_tool_name,
    _upload_to_gcs,
    generate_combined_html_report,
    generate_combined_json_report,
    generate_combined_report_from_dir,
    generate_html_report,
    run_all_evals,
)


@patch("cxas_scrapi.utils.reporting.gcs_utils.GCSUtils")
def test_upload_to_gcs_success(mock_gcs_cls: typing.Any) -> None:
    mock_gcs = mock_gcs_cls.return_value
    mock_gcs.upload_string.return_value = (
        "https://storage.mtls.cloud.google.com/bucket/report.html"
    )

    res = _upload_to_gcs("gs://bucket/report.html", "<html></html>")
    assert res == "https://storage.mtls.cloud.google.com/bucket/report.html"


@patch("cxas_scrapi.utils.reporting.gcs_utils.GCSUtils")
def test_upload_to_gcs_failure(mock_gcs_cls: typing.Any) -> None:
    mock_gcs = mock_gcs_cls.return_value
    mock_gcs.upload_string.side_effect = Exception("error")

    res = _upload_to_gcs("gs://bucket/report.html", "<html></html>")
    assert res is None


@patch("cxas_scrapi.utils.reporting._get_html_head")
@patch("cxas_scrapi.utils.reporting._upload_to_gcs")
@patch("builtins.open", new_callable=mock_open)
def test_generate_html_report_gcs_success(
    mock_file: typing.Any,
    mock_upload: typing.Any,
    mock_get_html_head: typing.Any,
) -> None:
    mock_get_html_head.return_value = "<html><head></head><body>"
    mock_upload.return_value = "https://url"
    results = [{"name": "test", "passed": True, "run": 1}]

    generate_html_report(results, "gs://bucket/report.html", "text", "model")

    mock_upload.assert_called_once()
    mock_file.assert_not_called()


@patch("cxas_scrapi.utils.reporting._get_html_head")
@patch("cxas_scrapi.utils.reporting._upload_to_gcs")
@patch("builtins.open", new_callable=mock_open)
def test_generate_html_report_gcs_fallback_with_extension(
    mock_file: typing.Any,
    mock_upload: typing.Any,
    mock_get_html_head: typing.Any,
) -> None:
    mock_get_html_head.return_value = "<html><head></head><body>"
    mock_upload.return_value = None
    results = [{"name": "test", "passed": True, "run": 1}]

    generate_html_report(
        results, "gs://bucket/fail_report.html", "text", "model"
    )

    mock_upload.assert_called_once()
    mock_file.assert_called_once_with("fail_report.html", "w")


@patch("cxas_scrapi.utils.reporting._get_html_head")
@patch("cxas_scrapi.utils.reporting._upload_to_gcs")
@patch("builtins.open", new_callable=mock_open)
def test_generate_html_report_gcs_fallback_no_extension(
    mock_file: typing.Any,
    mock_upload: typing.Any,
    mock_get_html_head: typing.Any,
) -> None:
    mock_get_html_head.return_value = "<html><head></head><body>"
    mock_upload.return_value = None
    results = [{"name": "test", "passed": True, "run": 1}]

    # Path with no extension
    generate_html_report(results, "gs://bucket/no_ext", "text", "model")

    mock_upload.assert_called_once()
    mock_file.assert_called_once_with("report_fallback.html", "w")


@patch("cxas_scrapi.utils.reporting._get_html_head")
@patch("cxas_scrapi.utils.reporting.tools.Tools")
@patch("builtins.open", new_callable=mock_open)
def test_generate_html_report_tools_failure(
    mock_file: typing.Any,
    mock_tools_cls: typing.Any,
    mock_get_html_head: typing.Any,
) -> None:
    mock_get_html_head.return_value = "<html><head></head><body>"
    # Simulate Tools(app_name).get_tools_map() failing
    mock_tools_cls.return_value.get_tools_map.side_effect = Exception(
        "Tools failed"
    )

    results = [{"name": "test", "passed": True, "run": 1}]
    generate_html_report(
        results, "local.html", "text", "model", app_name="projects/p"
    )

    mock_file.assert_called_once_with("local.html", "w")


@patch("cxas_scrapi.utils.reporting._get_html_head")
@patch("builtins.open", new_callable=mock_open)
def test_generate_html_report_local(
    mock_file: typing.Any, mock_get_html_head: typing.Any
) -> None:
    mock_get_html_head.return_value = "<html><head></head><body>"
    results = [
        {
            "name": "test_eval",
            "passed": False,
            "error": "Timeout",
            "run": 1,
            "session_id": "sess123",
            "turns": 5,
            "detailed_trace": ["User: hello", "Agent Text: hi"],
            "step_details": [
                {
                    "goal": "g",
                    "status": "Completed",
                    "success_criteria": "c",
                    "justification": "j",
                }
            ],
            "expectation_details": [
                {"expectation": "e", "status": "Met", "justification": "j2"}
            ],
        }
    ]

    generate_html_report(
        results=results,
        output_path="local.html",
        modality="audio",
        model="gemini-3.1-pro-preview",
        app_name="projects/p1/locations/l1/apps/a1",
        wall_clock_s=120.5,
    )

    mock_file.assert_called_once_with("local.html", "w")
    content = mock_file().write.call_args[0][0]
    assert "Simulation Eval Report" in content
    assert "0.0%" in content
    assert "audio" in content
    assert "gemini-3.1-pro-preview" in content
    assert "2.0m" in content
    assert "test_eval" in content
    assert "Timeout" in content
    assert "sess123" in content


def test_fmt_duration() -> None:
    assert _fmt_duration(None) == ""
    assert _fmt_duration(30) == "30.0s"
    assert _fmt_duration(90) == "1.5m"


def test_escape() -> None:
    assert _escape('<script>&"') == "&lt;script&gt;&amp;&quot;"


def test_resolve_tool_name() -> None:
    tools_map = {"projects/p/tools/t1": "MyTool"}
    assert _resolve_tool_name("projects/p/tools/t1", tools_map) == "MyTool"
    assert _resolve_tool_name("projects/p/tools/t2", tools_map) == "t2"
    assert _resolve_tool_name(None, tools_map) is None


def test_format_trace_line() -> None:
    tools_map = {"path/to/tool": "GreatTool"}
    line = "Tool Call: path/to/tool with args {}"
    assert "GreatTool" in _format_trace_line(line, tools_map)
    assert "Unrelated" in _format_trace_line("Unrelated", tools_map)


def test_generate_combined_html_report(tmp_path: typing.Any) -> None:
    output_path = os.path.join(tmp_path, "report.html")

    golden_results = [
        {
            "name": "test_golden",
            "passed": True,
            "turns": [
                {
                    "index": 1,
                    "semantic_score": 4,
                    "comparisons": [
                        {
                            "outcome": "PASS",
                            "type": "text",
                            "expected": "hello",
                            "actual": "hello",
                        }
                    ],
                }
            ],
            "expectations": [],
            "session_id": "sess_1",
            "session_parameters": {},
            "duration_s": 1.0,
        }
    ]

    sim_results = [
        {
            "name": "test_sim",
            "passed": True,
            "run": 1,
            "duration_s": 2.0,
            "goals": 1,
            "expectations": 0,
            "turns": 1,
            "session_id": "sess_2",
            "session_parameters": {},
            "step_details": [
                {
                    "goal": "test goal",
                    "success_criteria": "test criteria",
                    "status": "Completed",
                    "justification": "done",
                }
            ],
            "expectation_details": [],
            "detailed_trace": ["User: hi", "Agent Text: hello"],
        }
    ]

    tool_results = [
        {
            "name": "test_tool",
            "tool": "my_tool",
            "passed": True,
            "status": "PASSED",
            "latency_ms": 50,
            "errors": "",
        }
    ]

    callback_results = [
        {
            "name": "test_callback",
            "agent": "my_agent",
            "callback_type": "my_callback",
            "passed": True,
            "status": "PASSED",
            "error": "",
        }
    ]

    generate_combined_html_report(
        golden_results=golden_results,
        sim_results=sim_results,
        tool_results=tool_results,
        callback_results=callback_results,
        output_path=output_path,
        app_name="projects/test-proj/locations/global/apps/test-app",
    )

    assert os.path.exists(output_path)
    with open(output_path) as f:
        content = f.read()
        assert "Combined Eval Report" in content
        assert "test_golden" in content
        assert "test_sim" in content
        assert "test_tool" in content
        assert "test_callback" in content


def test_generate_combined_html_report_renders_naturalness(
    tmp_path: typing.Any,
) -> None:
    output_path = str(tmp_path / "report.html")
    sim_results = [
        {
            "name": "graded_sim",
            "run": 1,
            "passed": True,
            "goals": "1/1",
            "expectations": "0/0",
            "turns": 2,
            "duration_s": 1.0,
            "transcript": "User: hi\nAgent: hey there",
            "naturalness": "4.1/5",
            "naturalness_label": "Human-like",
            "naturalness_details": {
                "overall_score": 4.1,
                "overall_label": "Human-like",
                "turn_count": 1,
                "summary": "Sounds like a person.",
                "factor_averages": {"emotion": 4.0, "pacing": 4.2},
                "label_counts": {
                    "Bot-like": 0,
                    "Transitional": 0,
                    "Human-like": 1,
                },
                "turns": [
                    {
                        "turn_index": 0,
                        "agent_utterance": "hey there",
                        "label": "Human-like",
                        "score": 4.1,
                        "justification": "Warm and unscripted.",
                        "factors": [
                            {
                                "quality": "emotion",
                                "score": 4,
                                "value": "warm",
                                "reason": "Matched the caller's mood.",
                            },
                            {
                                "quality": "perceivedLatency",
                                "score": 1,
                                "value": "5.2s",
                                "reason": "unacceptable: 5.2s is slow",
                            },
                        ],
                    }
                ],
                "conversation_factors": [],
                "latency_ms_by_turn": {"0": 5200.0},
                "failure_reasons": ["perceived latency reached 5.2s on turn 0"],
            },
        },
        {"name": "ungraded_sim", "run": 1, "passed": True, "turns": 1},
    ]

    generate_combined_html_report(
        sim_results=sim_results,
        output_path=output_path,
        app_name="projects/test-proj/locations/global/apps/test-app",
    )

    with open(output_path) as f:
        content = f.read()

    assert "Human-like" in content
    assert "4.1/5" in content
    assert "Warm and unscripted." in content
    # The measured latency dimension and the hard failure it caused.
    assert "perceivedLatency" in content
    assert "5.2s" in content
    assert "perceived latency reached 5.2s on turn 0" in content
    # The ungraded run must still render, without a naturalness block.
    assert "ungraded_sim" in content


@patch("cxas_scrapi.utils.reporting._upload_to_gcs")
def test_generate_combined_html_report_gcs_success(
    mock_upload: typing.Any,
) -> None:
    mock_upload.return_value = "https://url"

    resolved_path = generate_combined_html_report(
        golden_results=[],
        sim_results=[],
        tool_results=[],
        callback_results=[],
        output_path="gs://bucket/report.html",
        app_name="projects/test-proj",
    )

    assert resolved_path == "https://url"
    mock_upload.assert_called_once()


@patch("cxas_scrapi.utils.reporting._upload_to_gcs")
@patch("builtins.open", new_callable=mock_open)
def test_generate_combined_html_report_gcs_fallback(
    mock_file: typing.Any, mock_upload: typing.Any
) -> None:
    mock_upload.return_value = None

    resolved_path = generate_combined_html_report(
        golden_results=[],
        sim_results=[],
        tool_results=[],
        callback_results=[],
        output_path="gs://bucket/report.html",
        app_name="projects/test-proj",
    )

    assert resolved_path == "report.html"
    mock_file.assert_any_call("report.html", "w")


def test_generate_combined_report_from_dir(tmp_path: typing.Any) -> None:
    evals_dir = tmp_path / "evals"
    evals_dir.mkdir()

    # Create dummy files
    sim_file = evals_dir / "sim_results.json"
    sim_file.write_text(json.dumps([{"name": "test_sim", "passed": True}]))

    tool_file = evals_dir / "tool_results.csv"
    df_tool = pd.DataFrame(
        [
            {
                "test_name": "test_tool",
                "tool": "my_tool",
                "status": "PASSED",
                "latency (ms)": 50,
                "errors": "",
            }
        ]
    )
    df_tool.to_csv(tool_file, index=False)

    callback_file = evals_dir / "callback_results.csv"
    df_callback = pd.DataFrame(
        [
            {
                "test_name": "test_callback",
                "agent_name": "my_agent",
                "callback_type": "my_callback",
                "status": "PASSED",
                "error_message": "",
            }
        ]
    )
    df_callback.to_csv(callback_file, index=False)

    output_path = evals_dir / COMBINED_REPORT_FILENAME

    generate_combined_report_from_dir(
        output_dir=str(evals_dir), output_path=str(output_path)
    )

    assert os.path.exists(output_path)
    with open(output_path) as f:
        content = f.read()
        assert "Combined Eval Report" in content
        assert "test_sim" in content
        assert "test_tool" in content
        assert "test_callback" in content


def test_generate_combined_report_from_dir_include_all(
    tmp_path: typing.Any,
) -> None:
    evals_dir = tmp_path / "evals"
    evals_dir.mkdir()

    # Create dummy files
    sim_file = evals_dir / "sim_results.json"
    sim_file.write_text(json.dumps([{"name": "test_sim", "passed": True}]))

    tool_file = evals_dir / "tool_results.csv"
    df_tool = pd.DataFrame(
        [
            {
                "test_name": "test_tool",
                "tool": "my_tool",
                "status": "PASSED",
                "latency (ms)": 50,
                "errors": "",
            }
        ]
    )
    df_tool.to_csv(tool_file, index=False)

    output_path = evals_dir / COMBINED_REPORT_FILENAME

    generate_combined_report_from_dir(
        output_dir=str(evals_dir), output_path=str(output_path), include=["all"]
    )

    assert os.path.exists(output_path)
    with open(output_path) as f:
        content = f.read()
        assert "test_sim" in content
        assert "test_tool" in content


def test_generate_combined_json_report_local(tmp_path: typing.Any) -> None:
    output_path = tmp_path / COMBINED_REPORT_JSON_FILENAME

    sim_results = [
        {
            "name": "test_sim",
            "passed": True,
            "detailed_trace": ["User: hi\nAgent Text: hello"],
            "_processed_trace": [("user", "hi")],
        },
        {"name": "test_sim", "passed": False},
    ]
    golden_results = [
        {
            "name": "test_golden",
            "passed": True,
            "turns": [{"semantic_score": 3.5}],
        }
    ]
    tool_results = [
        {
            "name": "test_tool",
            "tool": "my_tool",
            "passed": True,
            "status": "PASSED",
            "latency_ms": 50,
            "errors": "",
        }
    ]
    callback_results = [
        {
            "name": "test_callback",
            "agent": "my_agent",
            "callback_type": "my_callback",
            "passed": False,
            "status": "FAILED",
            "error": "boom",
        }
    ]

    resolved_path = generate_combined_json_report(
        golden_results=golden_results,
        sim_results=sim_results,
        tool_results=tool_results,
        callback_results=callback_results,
        output_path=str(output_path),
        app_name="projects/p/locations/l/apps/a",
        sim_wall_clock_s=12.5,
    )

    assert resolved_path == str(output_path)
    with open(output_path) as f:
        report = json.load(f)

    assert report["schema_version"] == 1
    assert report["app_name"] == "projects/p/locations/l/apps/a"
    assert report["sim_wall_clock_s"] == 12.5

    summary = report["summary"]
    assert summary["total"] == 5
    assert summary["passed"] == 3
    assert summary["pass_rate_pct"] == 60.0
    assert summary["simulation"] == {"total": 2, "passed": 1}
    assert summary["golden"] == {"total": 1, "passed": 1}
    assert summary["tool"] == {"total": 1, "passed": 1}
    assert summary["callback"] == {"total": 1, "passed": 0}

    results = report["results"]
    assert results["simulation"][0]["name"] == "test_sim"
    assert results["simulation"][0]["detailed_trace"] == [
        "User: hi\nAgent Text: hello"
    ]
    # Internal keys are stripped from the JSON output.
    assert "_processed_trace" not in results["simulation"][0]
    assert results["golden"][0]["turns"] == [{"semantic_score": 3.5}]
    assert results["tool"][0]["latency_ms"] == 50
    assert results["callback"][0]["error"] == "boom"


def test_generate_combined_json_report_serializes_non_json_values(
    tmp_path: typing.Any,
) -> None:
    output_path = tmp_path / "report.json"

    generate_combined_json_report(
        sim_results=[
            {
                "name": "test_sim",
                "passed": True,
                "started_at": datetime.datetime(2026, 7, 16, 12, 0, 0),
            }
        ],
        output_path=str(output_path),
    )

    with open(output_path) as f:
        report = json.load(f)
    assert "2026-07-16" in report["results"]["simulation"][0]["started_at"]


@patch("cxas_scrapi.utils.reporting._upload_to_gcs")
@patch("builtins.open", new_callable=mock_open)
def test_generate_combined_json_report_gcs_success(
    mock_file: typing.Any, mock_upload: typing.Any
) -> None:
    mock_upload.return_value = "https://url"

    resolved_path = generate_combined_json_report(
        sim_results=[{"name": "test_sim", "passed": True}],
        output_path="gs://bucket/report.json",
    )

    assert resolved_path == "https://url"
    mock_upload.assert_called_once_with(
        "gs://bucket/report.json", ANY, content_type="application/json"
    )
    mock_file.assert_not_called()


@patch("cxas_scrapi.utils.reporting._upload_to_gcs")
@patch("builtins.open", new_callable=mock_open)
def test_generate_combined_json_report_gcs_fallback(
    mock_file: typing.Any, mock_upload: typing.Any
) -> None:
    mock_upload.return_value = None

    resolved_path = generate_combined_json_report(
        sim_results=[{"name": "test_sim", "passed": True}],
        output_path="gs://bucket/report.json",
    )

    assert resolved_path == "report.json"
    mock_file.assert_any_call("report.json", "w")


@patch("cxas_scrapi.utils.reporting._upload_to_gcs")
@patch("builtins.open", new_callable=mock_open)
def test_generate_combined_json_report_gcs_fallback_no_extension(
    mock_file: typing.Any, mock_upload: typing.Any
) -> None:
    mock_upload.return_value = None

    resolved_path = generate_combined_json_report(
        sim_results=[{"name": "test_sim", "passed": True}],
        output_path="gs://bucket/no_ext",
    )

    assert resolved_path == "report_fallback.json"
    mock_file.assert_any_call("report_fallback.json", "w")


def test_generate_combined_report_from_dir_json(tmp_path: typing.Any) -> None:
    evals_dir = tmp_path / "evals"
    evals_dir.mkdir()

    sim_file = evals_dir / "sim_results.json"
    sim_file.write_text(json.dumps([{"name": "test_sim", "passed": True}]))

    tool_file = evals_dir / "tool_results.csv"
    df_tool = pd.DataFrame(
        [
            {
                "test_name": "test_tool",
                "tool": "my_tool",
                "status": "PASSED",
                "latency (ms)": 50,
                "errors": "",
            }
        ]
    )
    df_tool.to_csv(tool_file, index=False)

    callback_file = evals_dir / "callback_results.csv"
    df_callback = pd.DataFrame(
        [
            {
                "test_name": "test_callback",
                "agent_name": "my_agent",
                "callback_type": "my_callback",
                "status": "FAILED",
                "error_message": "boom",
            }
        ]
    )
    df_callback.to_csv(callback_file, index=False)

    resolved_path = generate_combined_report_from_dir(
        output_dir=str(evals_dir), report_format="json"
    )

    # Defaults to combined_report.json in the output directory.
    assert resolved_path == str(evals_dir / COMBINED_REPORT_JSON_FILENAME)
    assert os.path.exists(resolved_path)

    with open(resolved_path) as f:
        report = json.load(f)

    assert report["summary"]["total"] == 3
    assert report["summary"]["passed"] == 2
    assert report["results"]["simulation"][0]["name"] == "test_sim"
    assert report["results"]["tool"][0]["name"] == "test_tool"
    assert report["results"]["tool"][0]["passed"] is True
    assert report["results"]["callback"][0]["name"] == "test_callback"
    assert report["results"]["callback"][0]["passed"] is False


def test_generate_combined_report_from_dir_json_explicit_output(
    tmp_path: typing.Any,
) -> None:
    evals_dir = tmp_path / "evals"
    evals_dir.mkdir()

    sim_file = evals_dir / "sim_results.json"
    sim_file.write_text(json.dumps([{"name": "test_sim", "passed": True}]))

    output_path = tmp_path / "custom_report.json"

    resolved_path = generate_combined_report_from_dir(
        output_dir=str(evals_dir),
        output_path=str(output_path),
        report_format="json",
    )

    assert resolved_path == str(output_path)
    with open(output_path) as f:
        report = json.load(f)
    assert report["summary"]["total"] == 1


def test_generate_combined_report_from_dir_invalid_format(
    tmp_path: typing.Any,
) -> None:
    evals_dir = tmp_path / "evals"
    evals_dir.mkdir()

    with pytest.raises(ValueError, match="Unsupported report format"):
        generate_combined_report_from_dir(
            output_dir=str(evals_dir), report_format="xml"
        )


def test_generate_combined_report_from_dir_html_default_unchanged(
    tmp_path: typing.Any,
) -> None:
    evals_dir = tmp_path / "evals"
    evals_dir.mkdir()

    sim_file = evals_dir / "sim_results.json"
    sim_file.write_text(json.dumps([{"name": "test_sim", "passed": True}]))

    resolved_path = generate_combined_report_from_dir(output_dir=str(evals_dir))

    assert resolved_path == str(evals_dir / COMBINED_REPORT_FILENAME)
    with open(resolved_path) as f:
        assert "Combined Eval Report" in f.read()


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
@patch("glob.glob")
@patch("os.path.exists")
@patch("os.path.isdir")
def test_run_all_evals_filtering(
    mock_isdir: typing.Any,
    mock_exists: typing.Any,
    mock_glob: typing.Any,
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
) -> None:
    mock_exists.return_value = True
    mock_isdir.return_value = True
    mock_glob.side_effect = [
        ["evals/goldens/test1.yaml", "evals/goldens/test2.yaml"],
        ["evals/tool_tests/tool1.yaml"],
        ["evals/simulations/sim1.yaml"],
    ]

    # Mock load_golden_evals_from_yaml to return empty list
    mock_eval_utils.return_value.load_golden_evals_from_yaml.return_value = []

    run_all_evals(
        app_name="projects/p",
        filter_files=["test1.yaml"],
        goldens_dir="evals/goldens/",
        tool_test_file="evals/tool_tests/",
        simulation_dir="evals/simulations/",
    )

    mock_eval_utils.return_value.load_golden_evals_from_yaml.assert_called_once_with(
        "evals/goldens/test1.yaml"
    )


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
@patch("glob.glob")
@patch("os.path.exists")
@patch("os.path.isdir")
def test_run_all_evals_substring_filtering(
    mock_isdir: typing.Any,
    mock_exists: typing.Any,
    mock_glob: typing.Any,
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
) -> None:
    mock_exists.return_value = True
    mock_isdir.return_value = True
    mock_glob.side_effect = [
        ["evals/goldens/error.yaml", "evals/goldens/other.yaml"],
        ["evals/tool_tests/tool1.yaml"],
        ["evals/simulations/sim1.yaml"],
    ]

    # Mock load_golden_evals_from_yaml to return empty list
    mock_eval_utils.return_value.load_golden_evals_from_yaml.return_value = []

    run_all_evals(
        app_name="projects/p",
        filter_files=["ERROR"],
        goldens_dir="evals/goldens/",
        tool_test_file="evals/tool_tests/",
        simulation_dir="evals/simulations/",
    )

    mock_eval_utils.return_value.load_golden_evals_from_yaml.assert_called_once_with(
        "evals/goldens/error.yaml"
    )


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
@patch("glob.glob")
@patch("os.path.exists")
@patch("os.path.isdir")
@patch("cxas_scrapi.evals.runner.RunEvaluationOperationMetadata")
@patch("cxas_scrapi.utils.reporting.load_golden_results")
@patch("yaml.safe_load")
@patch("builtins.open", new_callable=mock_open)
def test_run_all_evals_tag_filtering(
    mock_open_file: typing.Any,
    mock_yaml_load: typing.Any,
    mock_load_golden: typing.Any,
    mock_proto: typing.Any,
    mock_isdir: typing.Any,
    mock_exists: typing.Any,
    mock_glob: typing.Any,
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
) -> None:
    mock_exists.return_value = True
    mock_isdir.return_value = True
    mock_glob.side_effect = [
        ["evals/goldens/test1.yaml"],
        ["evals/tool_tests/tool1.yaml"],
        ["evals/simulations/sim1.yaml"],
    ]

    # Mock load_golden_evals_from_yaml to return evaluations with tags
    mock_eval_utils.return_value.load_golden_evals_from_yaml.return_value = [
        {"name": "eval1", "tags": ["tag1"]},
        {"name": "eval2", "tags": ["tag2"]},
    ]

    # Mock yaml.safe_load to return simulations with tags
    mock_yaml_load.return_value = [
        {"name": "sim1", "tags": ["tag1"]},
        {"name": "sim2", "tags": ["tag2"]},
    ]

    mock_eval_client = mock_evaluations.return_value
    mock_eval_client.update_evaluation.return_value.name = "mock_name"

    run_all_evals(
        app_name="projects/p",
        filter_tags=["tag1"],
        goldens_dir="evals/goldens/",
        tool_test_file="evals/tool_tests/",
        simulation_dir="evals/simulations/",
    )

    # Verify that only eval1 was updated/run
    mock_eval_client.update_evaluation.assert_called_once_with(
        evaluation={"name": "eval1", "tags": ["tag1"]}, app_name="projects/p"
    )


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
@patch("glob.glob")
@patch("os.path.exists")
@patch("os.path.isdir")
@patch("yaml.safe_load")
@patch("builtins.open", new_callable=mock_open)
def test_run_all_evals_include_filtering(
    mock_open_file: typing.Any,
    mock_yaml_load: typing.Any,
    mock_isdir: typing.Any,
    mock_exists: typing.Any,
    mock_glob: typing.Any,
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
) -> None:
    mock_exists.return_value = True
    mock_isdir.return_value = True
    mock_glob.side_effect = [
        ["evals/simulations/sim1.yaml"],
    ]
    mock_yaml_load.return_value = [{"name": "sim1"}]

    # Call with ONLY sims
    run_all_evals(
        app_name="projects/p",
        include=["sims"],
        goldens_dir="evals/goldens/",
        tool_test_file="evals/tool_tests/",
        simulation_dir="evals/simulations/",
    )

    # Assert SimulationEvals was instantiated and run
    mock_sim_evals.assert_called_once_with(
        app_name="projects/p",
        rate_limiter=None,
        expectations_only=False,
        deployment_id=None,
        vertex_location="global",
        naturalness=None,
    )
    mock_sim_evals.return_value.run_simulations.assert_called_once()

    # Assert others were NOT called/instantiated
    mock_evaluations.assert_not_called()
    mock_tool_evals.assert_not_called()
    mock_callback_evals.assert_not_called()


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
@patch("glob.glob")
@patch("os.path.exists")
@patch("os.path.isdir")
def test_run_all_evals_include_tools(
    mock_isdir: typing.Any,
    mock_exists: typing.Any,
    mock_glob: typing.Any,
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
) -> None:
    mock_exists.return_value = True
    mock_isdir.return_value = True
    mock_glob.side_effect = [
        ["evals/tool_tests/tool1.yaml"],
    ]
    mock_tool_evals.return_value.load_tool_test_cases_from_file.return_value = [
        {"name": "case1"}
    ]

    # Call with ONLY tools
    run_all_evals(
        app_name="projects/p",
        include=["tools"],
        goldens_dir="evals/goldens/",
        tool_test_file="evals/tool_tests/",
        simulation_dir="evals/simulations/",
    )

    # Assert ToolEvals was instantiated and run
    mock_tool_evals.assert_called_once_with(app_name="projects/p")
    mock_tool_evals.return_value.run_tool_tests.assert_called_once()

    # Assert others were NOT called/instantiated
    mock_evaluations.assert_not_called()
    mock_sim_evals.assert_not_called()
    mock_callback_evals.assert_not_called()


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
@patch("glob.glob")
@patch("os.path.exists")
@patch("os.path.isdir")
def test_run_all_evals_include_callbacks(
    mock_isdir: typing.Any,
    mock_exists: typing.Any,
    mock_glob: typing.Any,
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
) -> None:
    mock_exists.return_value = True
    mock_isdir.return_value = True

    # Call with ONLY callbacks
    run_all_evals(
        app_name="projects/p",
        include=["callbacks"],
        goldens_dir="evals/goldens/",
        tool_test_file="evals/tool_tests/",
        simulation_dir="evals/simulations/",
    )

    # Assert CallbackEvals was instantiated and run
    mock_callback_evals.assert_called_once()
    mock_callback_evals.return_value.test_all_callbacks_in_app_dir.assert_called_once()

    # Assert others were NOT called/instantiated
    mock_evaluations.assert_not_called()
    mock_sim_evals.assert_not_called()
    mock_tool_evals.assert_not_called()


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
@patch("glob.glob")
@patch("os.path.exists")
@patch("os.path.isdir")
@patch(
    "builtins.open",
    new_callable=mock_open,
    read_data="evals:\n  - name: sim1\n    tags: [P0]",
)
def test_run_all_evals_dict_based_simulations(
    mock_file: typing.Any,
    mock_isdir: typing.Any,
    mock_exists: typing.Any,
    mock_glob: typing.Any,
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
) -> None:
    mock_exists.return_value = True
    mock_isdir.return_value = True
    mock_glob.side_effect = [
        ["evals/simulations/sims.yaml"],
    ]

    run_all_evals(
        app_name="projects/p",
        include=["sims"],
        simulation_dir="evals/simulations/",
    )

    # Verify SimulationEvals was instantiated and run
    mock_sim_evals.assert_called_once_with(
        app_name="projects/p",
        rate_limiter=None,
        expectations_only=False,
        deployment_id=None,
        vertex_location="global",
        naturalness=None,
    )
    mock_sim_evals.return_value.run_simulations.assert_called_once_with(
        [
            {
                "name": "sim1",
                "tags": ["P0"],
                "session_parameters": {},
                "expectations": [],
            }
        ],
        runs=1,
        parallel=1,
        sim_user_model=None,
        eval_model=None,
        modality="text",
        background_noise_file=None,
        burst_noise_files=None,
        use_tool_fakes=False,
        skip_playback_wait=False,
        single_bidi_stream=False,
        progress_callback=ANY,
        capture_agent_audio=False,
        naturalness=None,
    )


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
@patch("glob.glob")
@patch("os.path.exists")
@patch("os.path.isdir")
@patch("yaml.safe_load")
@patch("builtins.open", new_callable=mock_open)
def test_run_all_evals_with_deployment_id(
    mock_open_file: typing.Any,
    mock_yaml_load: typing.Any,
    mock_isdir: typing.Any,
    mock_exists: typing.Any,
    mock_glob: typing.Any,
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
) -> None:
    mock_exists.return_value = True
    mock_isdir.return_value = True
    mock_glob.side_effect = [
        ["evals/simulations/sim1.yaml"],
    ]
    mock_yaml_load.return_value = [{"name": "sim1"}]

    run_all_evals(
        app_name="projects/p",
        include=["sims"],
        simulation_dir="evals/simulations/",
        deployment_id="dep123",
    )

    mock_sim_evals.assert_called_once_with(
        app_name="projects/p",
        rate_limiter=None,
        expectations_only=False,
        deployment_id="dep123",
        vertex_location="global",
        naturalness=None,
    )


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
@patch("glob.glob")
@patch("os.path.exists")
@patch("os.path.isdir")
@patch("yaml.safe_load")
@patch("builtins.open", new_callable=mock_open)
def test_run_all_evals_custom_vertex_location(
    mock_open_file: typing.Any,
    mock_yaml_load: typing.Any,
    mock_isdir: typing.Any,
    mock_exists: typing.Any,
    mock_glob: typing.Any,
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
) -> None:
    mock_exists.return_value = True
    mock_isdir.return_value = True
    mock_glob.side_effect = [
        ["evals/simulations/sim1.yaml"],
    ]
    mock_yaml_load.return_value = [{"name": "sim1"}]

    run_all_evals(
        app_name="projects/p",
        include=["sims"],
        simulation_dir="evals/simulations/",
        vertex_location="europe-west1",
    )

    mock_sim_evals.assert_called_once_with(
        app_name="projects/p",
        rate_limiter=None,
        expectations_only=False,
        deployment_id=None,
        vertex_location="europe-west1",
        naturalness=None,
    )


def test_load_sim_test_cases_merges_common_parameters() -> None:
    yaml_data = """
common_session_parameters:
  disable_disclaimer: true
  originationNumber: "1234567890"
evals:
  - name: sim1
    session_parameters:
      custom_param: "hello"
  - name: sim2
"""
    with patch("builtins.open", mock_open(read_data=yaml_data)):
        cases = _load_sim_test_cases("dummy.yaml")

        assert len(cases) == 2
        # Case 1 should have merged common params and its own params
        assert cases[0]["session_parameters"] == {
            "disable_disclaimer": True,
            "originationNumber": "1234567890",
            "custom_param": "hello",
        }
        # Case 2 should just have common params
        assert cases[1]["session_parameters"] == {
            "disable_disclaimer": True,
            "originationNumber": "1234567890",
        }


def test_load_sim_test_cases_merges_common_expectations() -> None:
    yaml_data = """
common_expectations:
  - "The agent welcomes the user"
  - "The agent behaves politely"
evals:
  - name: sim1
    expectations:
      - "The agent offers options"
  - name: sim2
"""
    with patch("builtins.open", mock_open(read_data=yaml_data)):
        cases = _load_sim_test_cases("dummy.yaml")

        assert len(cases) == 2
        # Case 1 should have merged expectations
        assert cases[0]["expectations"] == [
            "The agent welcomes the user",
            "The agent behaves politely",
            "The agent offers options",
        ]
        # Case 2 should just have common expectations
        assert cases[1]["expectations"] == [
            "The agent welcomes the user",
            "The agent behaves politely",
        ]


def test_load_sim_test_cases_without_common_naturalness() -> None:
    """Existing sim files must not grow a naturalness key."""
    yaml_data = """
evals:
  - name: sim1
"""
    with patch("builtins.open", mock_open(read_data=yaml_data)):
        cases = _load_sim_test_cases("dummy.yaml")

    assert "naturalness_metric" not in cases[0]


def test_load_sim_test_cases_merges_common_naturalness_metric() -> None:
    yaml_data = """
common_naturalness_metric:
  pass_threshold: 3.5
  turn_weight: 0.8
evals:
  - name: inherits
  - name: overrides
    naturalness_metric:
      pass_threshold: 4.0
  - name: opts_out
    naturalness_metric: false
"""
    with patch("builtins.open", mock_open(read_data=yaml_data)):
        cases = _load_sim_test_cases("dummy.yaml")

    assert cases[0]["naturalness_metric"] == {
        "pass_threshold": 3.5,
        "turn_weight": 0.8,
    }
    # A per-eval mapping wins key-by-key.
    assert cases[1]["naturalness_metric"] == {
        "pass_threshold": 4.0,
        "turn_weight": 0.8,
    }
    # An explicit false opts a single eval out of the file-wide metric.
    assert cases[2]["naturalness_metric"] is False


def test_generate_combined_report_from_dir_timestamped(
    tmp_path: typing.Any,
) -> None:
    evals_dir = tmp_path / "evals"
    evals_dir.mkdir()

    # Create dummy files with timestamp
    timestamp = "20260622_171403"
    sim_file = evals_dir / f"sim_results_{timestamp}.json"
    sim_file.write_text(json.dumps([{"name": "test_sim", "passed": True}]))

    tool_file = evals_dir / f"tool_results_{timestamp}.csv"
    df_tool = pd.DataFrame(
        [
            {
                "test_name": "test_tool",
                "tool": "my_tool",
                "status": "PASSED",
                "latency (ms)": 50,
                "errors": "",
            }
        ]
    )
    df_tool.to_csv(tool_file, index=False)

    callback_file = evals_dir / f"callback_results_{timestamp}.csv"
    df_callback = pd.DataFrame(
        [
            {
                "test_name": "test_callback",
                "agent_name": "my_agent",
                "callback_type": "my_callback",
                "status": "PASSED",
                "error_message": "",
            }
        ]
    )
    df_callback.to_csv(callback_file, index=False)

    # Call generate_combined_report_from_dir without output_path (so it
    # resolves it) and with run=False (so it loads from files)
    resolved_path = generate_combined_report_from_dir(
        output_dir=str(evals_dir),
        run=False,
    )

    expected_output_path = os.path.join(
        str(evals_dir),
        add_timestamp_suffix(COMBINED_REPORT_FILENAME, timestamp),
    )
    assert resolved_path == expected_output_path
    assert os.path.exists(expected_output_path)

    with open(expected_output_path) as f:
        content = f.read()
        assert "Combined Eval Report" in content
        assert "test_sim" in content
        assert "test_tool" in content
        assert "test_callback" in content


@patch("cxas_scrapi.evals.runner.Evaluations")
@patch("cxas_scrapi.evals.runner.ToolEvals")
@patch("cxas_scrapi.evals.runner.SimulationEvals")
@patch("cxas_scrapi.evals.runner.CallbackEvals")
@patch("cxas_scrapi.evals.runner.EvalUtils")
def test_run_all_evals_writes_timestamped_files(
    mock_eval_utils: typing.Any,
    mock_callback_evals: typing.Any,
    mock_sim_evals: typing.Any,
    mock_tool_evals: typing.Any,
    mock_evaluations: typing.Any,
    tmp_path: typing.Any,
) -> None:
    # Set up real files in tmp_path
    sims_dir = tmp_path / "simulations"
    sims_dir.mkdir()
    sim_yaml = sims_dir / "sims.yaml"
    sim_yaml.write_text("evals:\n  - name: sim1\n    tags: [P0]")

    output_dir = tmp_path / "output"
    output_dir.mkdir()

    # Mock SimulationEvals.run_simulations to return dummy results
    mock_sim_evals.return_value.run_simulations.return_value = [
        {"name": "sim1", "passed": True}
    ]

    timestamp = "20260622_171403"

    run_all_evals(
        app_name="projects/p",
        include=["sims"],
        simulation_dir=str(sims_dir),
        output_dir=str(output_dir),
        timestamp=timestamp,
    )

    # Verify file was written with timestamp
    expected_file = output_dir / f"sim_results_{timestamp}.json"
    assert expected_file.exists()

    with open(expected_file) as f:
        data = json.load(f)
        assert len(data) == 1
        assert data[0]["name"] == "sim1"
        assert data[0]["passed"] is True


@patch("cxas_scrapi.utils.reporting.evals_runner.run_all_evals")
def test_run_all_evals_expectations_only(
    mock_run_all_evals: typing.Any,
) -> None:
    run_all_evals(
        app_name="projects/p",
        expectations_only=True,
    )
    mock_run_all_evals.assert_called_once_with(
        app_name="projects/p",
        modality="text",
        sim_user_model=None,
        eval_model=None,
        runs=1,
        goldens_dir=None,
        tool_test_file=None,
        simulation_dir=None,
        app_dir=None,
        output_dir=None,
        filter_files=None,
        filter_tags=None,
        filter_names=None,
        parallel=1,
        golden_parallel=1,
        golden_timeout=600,
        include=None,
        bg_noise_file=None,
        burst_noise_files=None,
        use_tool_fakes=False,
        timestamp=None,
        expectations_only=True,
        deployment_id=None,
        skip_playback_wait=False,
        single_bidi_stream=False,
        progress_callback=None,
        capture_agent_audio=False,
        vertex_location="global",
        naturalness=None,
    )


@pytest.fixture
def sim_runner() -> typing.Any:
    scripts_dir = os.path.abspath(".agents/skills/cxas-agent-foundry/scripts")
    if scripts_dir not in sys.path:
        sys.path.insert(0, scripts_dir)

    resolve_patch = patch(
        "config.resolve_project_dir", return_value="/dummy/project"
    )
    path_patch = patch(
        "config.get_project_path",
        side_effect=lambda *parts: os.path.join("/dummy/project", *parts),
    )
    with resolve_patch, path_patch:
        spec = importlib.util.spec_from_file_location(
            "scrapi_sim_runner",
            os.path.join(scripts_dir, "scrapi-sim-runner.py"),
        )
        runner = importlib.util.module_from_spec(spec)
        sys.modules["scrapi_sim_runner"] = runner
        spec.loader.exec_module(runner)
    return runner


def test_sim_runner_load_sim_templates_merges_common_expectations(
    sim_runner: typing.Any,
) -> None:
    yaml_data = """
common_expectations:
  - "Common expectation 1"
  - "Common expectation 2"
evals:
  - name: sim1
    expectations:
      - "Specific expectation 1"
  - name: sim2
"""
    with patch("os.path.exists", return_value=True):  # noqa: SIM117
        with patch("builtins.open", mock_open(read_data=yaml_data)):
            templates = sim_runner.load_sim_templates()

            assert len(templates) == 2
            assert templates["sim1"]["expectations"] == [
                "Specific expectation 1",
                "Common expectation 1",
                "Common expectation 2",
            ]
            assert templates["sim2"]["expectations"] == [
                "Common expectation 1",
                "Common expectation 2",
            ]


def test_sim_runner_load_sim_templates_handles_missing_common_expectations(
    sim_runner: typing.Any,
) -> None:
    yaml_data = """
evals:
  - name: sim1
    expectations:
      - "Specific expectation 1"
"""
    with patch("os.path.exists", return_value=True):  # noqa: SIM117
        with patch("builtins.open", mock_open(read_data=yaml_data)):
            templates = sim_runner.load_sim_templates()

            assert len(templates) == 1
            assert templates["sim1"]["expectations"] == [
                "Specific expectation 1"
            ]


def test_sim_runner_load_sim_templates_backward_compatibility_with_list(
    sim_runner: typing.Any,
) -> None:
    yaml_data = """
- name: sim1
  expectations:
    - "Specific expectation 1"
- name: sim2
  expectations:
    - "Specific expectation 2"
"""
    with patch("os.path.exists", return_value=True):  # noqa: SIM117
        with patch("builtins.open", mock_open(read_data=yaml_data)):
            templates = sim_runner.load_sim_templates()

            assert len(templates) == 2
            assert templates["sim1"]["expectations"] == [
                "Specific expectation 1"
            ]
            assert templates["sim2"]["expectations"] == [
                "Specific expectation 2"
            ]


def _make_sample_turn_trace(
    with_filler: bool = False,
) -> dict[str, typing.Any]:
    child_spans = [
        {
            "name": "VAD",
            "startTime": "2026-10-02T14:00:00.000Z",
            "endTime": "2026-10-02T14:00:01.000Z",
            "duration": "1.000s",
            "attributes": {
                "user transcript": "My internet is down",
            },
        },
        {
            "name": "AGENT: Troubleshoting_Agent",
            "startTime": "2026-10-02T14:00:01.000Z",
            "endTime": "2026-10-02T14:00:03.500Z",
            "duration": "2.500s",
            "childSpans": [
                {
                    "name": "CALLBACK: before_model_callback",
                    "startTime": "2026-10-02T14:00:01.000Z",
                    "endTime": "2026-10-02T14:00:01.180Z",
                    "duration": "0.180s",
                    "attributes": {
                        "[debug] detailed latency": {
                            "total code execution latency (ms)": 60,
                            "waiting for external calls (ms)": 0,
                            "sandbox init latency (ms)": 5,
                        }
                    },
                },
                {
                    "name": "CALLBACK: before_model_callback",
                    "startTime": "2026-10-02T14:00:01.180Z",
                    "endTime": "2026-10-02T14:00:01.300Z",
                    "duration": "0.120s",
                    "attributes": {
                        "[debug] detailed latency": {
                            "total code execution latency (ms)": 40,
                            "waiting for external calls (ms)": 0,
                            "sandbox init latency (ms)": 0,
                        }
                    },
                },
                {
                    "name": "LLM",
                    "startTime": "2026-10-02T14:00:01.300Z",
                    "endTime": "2026-10-02T14:00:02.100Z",
                    "duration": "0.800s",
                    "attributes": {
                        "model": "gemini-2.5-flash-001",
                        "time to first token (ms)": 450,
                        "time to first audio (ms)": 0 if with_filler else 700,
                    },
                },
                {
                    "name": "TOOL: verify_equipment_status",
                    "startTime": "2026-10-02T14:00:01.800Z",
                    "endTime": "2026-10-02T14:00:03.000Z",
                    "duration": "1.200s",
                    "attributes": {
                        "toolDisplayName": "verify_equipment_status",
                        "time to first audio (ms)": 150 if with_filler else 0,
                    },
                },
                {
                    "name": "LLM",
                    "startTime": "2026-10-02T14:00:03.000Z",
                    "endTime": "2026-10-02T14:00:03.500Z",
                    "duration": "0.500s",
                    "attributes": {
                        "model": "gemini-2.5-flash-001",
                        "time to first token (ms)": 220,
                        "time to first audio (ms)": 400 if with_filler else 0,
                    },
                },
            ],
        },
    ]
    return {
        "messages": [
            {"role": "user", "chunks": [{"text": "My internet is down"}]},
            {
                "role": "agent",
                "chunks": [{"text": "Let me check your equipment right now."}],
            },
        ],
        "rootSpan": {
            "name": "TURN",
            "startTime": "2026-10-02T14:00:00.000Z",
            "endTime": "2026-10-02T14:00:03.500Z",
            "duration": "3.500s",
            "childSpans": child_spans,
        },
    }


def test_perceived_latency_rendering_in_html_reports(
    tmp_path: typing.Any,
) -> None:
    # Create an agent JSON definition to verify Option 1 callback folder resolution
    agent_dir = tmp_path / "agents" / "Troubleshoting_Agent"
    agent_dir.mkdir(parents=True)
    agent_json = agent_dir / "Troubleshoting_Agent.json"
    agent_json.write_text(
        json.dumps(
            {
                "displayName": "Troubleshoting_Agent",
                "beforeModelCallbacks": [
                    {
                        "description": " Callback 1",
                        "pythonCode": "agents/Troubleshoting_Agent/before_model_callbacks/before_model_callbacks_01/python_code.py",
                    },
                    {
                        "description": "Disconnect Callback",
                        "pythonCode": "agents/Troubleshoting_Agent/before_model_callbacks/before_model_callbacks_disconnect/python_code.py",
                    },
                ],
            }
        )
    )

    turn_trace = _make_sample_turn_trace(with_filler=True)
    sim_results = [
        {
            "name": "test_voice_sim",
            "passed": True,
            "run": 1,
            "duration_s": 3.5,
            "goals": "1/1",
            "expectations": "1/1",
            "turns": 1,
            "session_id": "sess_pl_1",
            "detailed_trace": [
                "User: My internet is down",
                "Tool Call: verify_equipment_status",
                "Tool Response: {'status': 'online'}",
                "Agent Text: Let me check your equipment right now.",
                "[Callback: BeforeModel (Troubleshoting_Agent)] (180.0 ms)",
                "[Callback: BeforeModel (Troubleshoting_Agent)] (120.0 ms)",
            ],
            "turn_traces": [turn_trace],
            "step_details": [],
            "expectation_details": [],
        }
    ]

    sim_report_path = os.path.join(tmp_path, "sim_report.html")
    generate_html_report(
        results=sim_results,
        output_path=sim_report_path,
        modality="audio",
        model="gemini-2.5-flash",
    )
    with open(sim_report_path) as f:
        sim_html = f.read()

    # Verify Suite, Conversation, and Turn drawers exist and are collapsed by default
    assert 'class="pl-suite-details latency-drawer"' in sim_html
    assert '<span class="pl-arrow">&#9654;</span>' in sim_html
    assert 'class="pl-conv-details latency-drawer"' in sim_html
    assert "pl-turn-details" in sim_html
    assert "wf-first-audio-line" in sim_html
    assert "toggleAllLatency()" in sim_html
    assert "verify_equipment_status" in sim_html
    # Verify Option 1 callback folder names (without python_code.py) and collapsible views
    assert "before_model_callbacks_01" in sim_html
    assert "before_model_callbacks_disconnect" in sim_html
    assert "python_code.py" not in sim_html
    assert "cb-sub-details" in sim_html
    assert "wf-cb-details" in sim_html
    assert "<th>Observation</th>" in sim_html
    assert "Actionable Guidance" not in sim_html
    assert "Callback Time Decomposition" in sim_html
    assert "code 60ms + sandbox 120ms" in sim_html
    assert "cb-spans-details" in sim_html
    assert "Callback Spans (2)" in sim_html
    assert "Customer Speech (VAD)" not in sim_html
    assert "wf-fa-pin" in sim_html
    assert "Audio Streaming" in sim_html
    assert "pl-stack-legend" in sim_html
    assert "pl-legend-dot" in sim_html

    # Verify combined HTML report also includes all three collapsed latency drawers
    combined_report_path = os.path.join(tmp_path, "combined_report.html")
    generate_combined_html_report(
        golden_results=[],
        sim_results=sim_results,
        tool_results=[],
        callback_results=[],
        output_path=combined_report_path,
        sim_modality="audio",
    )
    with open(combined_report_path) as f:
        combined_html = f.read()

    assert 'class="pl-suite-details latency-drawer"' in combined_html
    assert '<span class="pl-arrow">&#9654;</span>' in combined_html
    assert 'class="pl-conv-details latency-drawer"' in combined_html
    assert "pl-turn-details" in combined_html
    assert "wf-first-audio-line" in combined_html
    assert "before_model_callbacks_01" in combined_html
    assert "python_code.py" not in combined_html
    assert "Callback Time Decomposition" in combined_html
    assert "cb-spans-details" in combined_html
    assert "Callback Spans (2)" in combined_html


def test_perceived_latency_standalone_tts_and_barge_in_fallback() -> None:
    # 1. Cascaded turn with standalone TTS span
    cascaded_turn = {
        "messages": [
            {"role": "user", "chunks": [{"text": "Schedule a technician"}]},
            {"role": "agent", "chunks": [{"text": "I can help with that."}]},
        ],
        "rootSpan": {
            "name": "root",
            "startTime": "2026-10-02T14:00:00.000Z",
            "endTime": "2026-10-02T14:00:03.000Z",
            "duration": "3.000s",
            "attributes": {
                "perceived latency (ms)": 1150,
                "input types": ["audio"],
            },
            "childSpans": [
                {
                    "name": "VAD",
                    "startTime": "2026-10-02T14:00:00.000Z",
                    "endTime": "2026-10-02T14:00:01.000Z",
                    "duration": "1.000s",
                    "attributes": {},
                },
                {
                    "name": "LLM",
                    "startTime": "2026-10-02T14:00:01.100Z",
                    "endTime": "2026-10-02T14:00:02.200Z",
                    "duration": "1.100s",
                    "attributes": {
                        "model": "gemini-2.5-flash",
                        "time to first chunk (ms)": 600,
                    },
                },
                {
                    "name": "TTS",
                    "startTime": "2026-10-02T14:00:01.900Z",
                    "endTime": "2026-10-02T14:00:02.400Z",
                    "duration": "0.500s",
                    "attributes": {
                        "model": "gemini-composite-v1",
                        "time to first audio (ms)": 250,
                        "audio duration (ms)": 1800,
                    },
                },
            ],
        },
    }
    res_cascaded = LatencyParser.parse_turn_perceived_latency(cascaded_turn, 1)
    assert res_cascaded is not None
    assert round(res_cascaded["pl_ms"]) == 1150
    assert round(res_cascaded["first_audio_rel_ms"]) == 1150
    assert all(w["kind"] != "vad" for w in res_cascaded["waterfall_spans"])
    assert res_cascaded["pl_source"] == "TTS (gemini-composite-v1)"
    bd = res_cascaded["breakdown_ms"]
    assert round(bd["llm_ttfc_ms"]) == 800
    assert round(bd["tts_ms"]) == 250

    # 2. Barged-in turn where TTS span is missing from childSpans
    barged_in_turn = {
        "messages": [
            {"role": "user", "chunks": [{"text": "Wait actually tomorrow"}]},
            {"role": "agent", "chunks": [{"text": "Checking tomorrow..."}]},
        ],
        "rootSpan": {
            "name": "root",
            "startTime": "2026-10-02T14:00:05.000Z",
            "endTime": "2026-10-02T14:00:08.000Z",
            "duration": "3.000s",
            "attributes": {
                "perceived latency (ms)": 1600,
                "agent audio uri": "gs://bucket/agent.wav",
                "input types": ["audio"],
            },
            "childSpans": [
                {
                    "name": "VAD",
                    "startTime": "2026-10-02T14:00:05.000Z",
                    "endTime": "2026-10-02T14:00:06.000Z",
                    "duration": "1.000s",
                    "attributes": {},
                },
                {
                    "name": "LLM",
                    "startTime": "2026-10-02T14:00:06.200Z",
                    "endTime": "2026-10-02T14:00:07.500Z",
                    "duration": "1.300s",
                    "attributes": {
                        "model": "gemini-2.5-flash",
                        "time to first chunk (ms)": 900,
                    },
                },
            ],
        },
    }
    res_barged = LatencyParser.parse_turn_perceived_latency(barged_in_turn, 2)
    assert res_barged is not None
    assert round(res_barged["pl_ms"]) == 1600
    assert res_barged["pl_source"] == "TTS (Barged-in)"
    bd_barged = res_barged["breakdown_ms"]
    assert round(bd_barged["llm_ttfc_ms"]) == 900
    assert round(bd_barged["tts_ms"]) == 500