{# pyre-ignore-all-errors #} {% macro render_trend_svg(chart) %} {% if not chart %}
No turn data available
{% else %} {% for g in chart.grid_lines %} {{ g.label }} {% endfor %} {% for s in chart.series %} {% for dot in s.dots %} {% endfor %} {{ s.last_val_fmt }} {% endfor %} Turns {{ chart.nmax }} {% for b in chart.bars %} {{ b.label }} {% endfor %} {% for h in chart.hits %} {{ h.title }} {% endfor %} {% endif %} {% endmacro %} {% macro render_suite_latency(suite_pl, trend_chart, _fmt_ms, _pl_status_cls) %} {% if suite_pl and suite_pl.get("primary") %} {% set prim = suite_pl["primary"] %} {% set slices = suite_pl.get("slices", {}) %} {% set active_key = suite_pl.get("primary_slice_key", "user_turns") %} {% set p50_cls = _pl_status_cls(prim.get("p50_ms")) %} {% set p90_cls = _pl_status_cls(prim.get("p90_ms")) %} {% set p50_m = _fmt_ms(prim.get("p50_ms")) %} {% set p90_m = _fmt_ms(prim.get("p90_ms")) %} {% set avg_m = _fmt_ms(prim.get("avg_ms")) %} {% set total_turns = (suite_pl.get("total_turns", 0) or 0) | int %} {% set conv_cnt = (suite_pl.get("conversations_count", 0) or 0) | int %} {% set mod_lbl = suite_pl.get("modality_label", "voice") or "voice" %}
▶⏱ Perceived Latency & Bottleneck Analysis (Click to expand pre-speech breakdown, component observations & filler signals) p50: {{ p50_m }} p90: {{ p90_m }} Mean: {{ avg_m }}
{{ "{:,}".format(total_turns) }} turns across {{ "{:,}".format(conv_cnt) }} evaluations · {{ mod_lbl }}
{% if suite_pl.get("kpi_tiles") %}
{% for tile in suite_pl.get("kpi_tiles", []) %}
{{ tile.get("label", "") }}
{{ tile.get("value", "") }}
{{ tile.get("meta", "") }}
{% endfor %}
{% endif %}

Perceived latency by turn

p50 p90 Turn volume
{{ render_trend_svg(trend_chart) }}
{% if suite_pl.get("metrics_tables") %}

Latency metrics

{% for mt in suite_pl.get("metrics_tables", []) %} {% if suite_pl.get("metrics_tables", []) | length > 1 %}

{{ mt.get("title", "") }}

{% endif %} {% for r in mt.get("rows", []) %} {% endfor %}
MetricTargetMeanp50p90p95MaxN
{{ r.get("metric", "") }} {{ r.get("target", "") }} {{ r.get("avg", "—") }} {{ r.get("p50", "—") }} {{ r.get("p90", "—") }} {{ r.get("p95", "—") }} {{ r.get("max", "—") }} {{ r.get("n", "—") }}
{% endfor %}
{% endif %}

Where a turn's time goes

{% if suite_pl.get("turn_parts") %} {% for p in suite_pl.get("turn_parts", []) %} {% set p_w = (p.get("pct_width", 0.0) or 0.0) | float %} {% set w_val = 100.0 if p_w > 100.0 else (0.5 if p_w < 0.5 else p_w) %}
{{ p.get("label", "") }} {{ p.get("dur_fmt", "") }}{% if p.get("pct_span_time") %} · {{ p.get("pct_span_time") }}{% endif %}
{% endfor %} {% else %}
No breakdown data
{% endif %}

Tool latency

{% if suite_pl.get("top_tools") %} {% for t in suite_pl.get("top_tools", []) %} {% endfor %} {% if suite_pl.get("overflow_summary") %} {% set overflow = suite_pl.get("overflow_summary") %} {% endif %}
ToolCallsp50p95MaxShare
{{ t.get("name", "") }} {{ t.get("calls", 0) }} {{ t.get("p50_fmt", "—") }} {{ t.get("p95_fmt", "—") }} {{ t.get("max_fmt", "—") }} {{ t.get("share_pct", "0%") }}
+ {{ overflow.get("remaining_count", 0) }} more tools ({{ overflow.get("share_pct", "") }} total tool time)
{% else %}
No tools invoked
{% endif %}
{% if suite_pl.get("slowest_evals") %}

Slowest evaluations

{% for se in suite_pl.get("slowest_evals", []) %} {% set e_nm = se.get("name", "") %} {% set e_type = se.get("eval_type", "sim") %} {% set jump_arg_type = "golden" if e_type == "golden" else "sim" %} {% endfor %}
EvaluationTurnsMedian turnSlowest turnTurn breakdown
{{ e_nm }} {{ e_type }} {{ se.get("turns_count", 0) }} {{ se.get("median_turn_fmt", "—") }} Turn {{ se.get("slowest_turn_num", 1) }} ({{ se.get("slowest_turn_fmt", "—") }}) {{ se.get("breakdown_str", "") }}
{% endif %}
Clock: VAD.endTime → First Audio (startTime + TTFA)
{% set slice_order = [("user_turns", "User Turns"), ("event_turns", "Event / System Turns"), ("all_turns", "All Turns")] %} {% for skey, s_default_label in slice_order %} {% set sdata = slices.get(skey) %} {% if sdata and sdata.get("total_turns", 0) > 0 %} {% set is_active = (skey == active_key) %} {% set btn_cls = "pl-slice-btn active" if is_active else "pl-slice-btn" %} {% set spoken_cnt = sdata.get("spoken_count", 0) %} {% set silent_cnt = sdata.get("silent_count", 0) %} {% set cnt_lbl = (spoken_cnt ~ " spoken, " ~ silent_cnt ~ " silent") if silent_cnt > 0 else (spoken_cnt ~ "") %} {% endif %} {% endfor %}
{% for skey, s_default_label in slice_order %} {% set sdata = slices.get(skey) %} {% if sdata and sdata.get("total_turns", 0) > 0 %} {% set is_active = (skey == active_key) %} {% set disp = "block" if is_active else "none" %} {% set st = sdata.get("stack", {}) %} {% set slo_under_2s = (100.0 - (sdata.get("over_2s_pct", 0.0) | float)) | round(1) %} {% set slo_cls = "pass" if slo_under_2s >= 80 else ("warn" if slo_under_2s >= 50 else "fail") %} {% set avg_c = _pl_status_cls(sdata.get("avg_ms")) %} {% set p50_c = _pl_status_cls(sdata.get("p50_ms")) %} {% set p90_c = _pl_status_cls(sdata.get("p90_ms")) %} {% set avg_s = _fmt_ms(sdata.get("avg_ms")) %} {% set unm_s = _fmt_ms(sdata.get("unmasked_avg_ms")) %} {% set p50_s = _fmt_ms(sdata.get("p50_ms")) %} {% set p90_s = _fmt_ms(sdata.get("p90_ms")) %} {% set max_s = _fmt_ms(sdata.get("max_ms")) %} {% set u1_pct = sdata.get("under_1s_pct", 0) %} {% set o3_pct = sdata.get("over_3s_pct", 0) %} {% set f_cnt = sdata.get("filler_masked_count", 0) %} {% set f_sav = _fmt_ms(sdata.get("filler_saved_avg_ms", 0)) %} {% set s_lbl = sdata.get("label", "") %} {% set cb_w = 0.0 if st.get("callback_pct", 0) < 0 else st.get("callback_pct", 0) %} {% set cb_m = _fmt_ms(st.get("callback_ms", 0)) %} {% set cb_p = st.get("callback_pct", 0) %} {% set tl_w = 0.0 if st.get("tool_pct", 0) < 0 else st.get("tool_pct", 0) %} {% set tl_m = _fmt_ms(st.get("tool_ms", 0)) %} {% set tl_p = st.get("tool_pct", 0) %} {% set lm_w = 0.0 if st.get("llm_ttfc_pct", 0) < 0 else st.get("llm_ttfc_pct", 0) %} {% set lm_m = _fmt_ms(st.get("llm_ttfc_ms", 0)) %} {% set lm_p = st.get("llm_ttfc_pct", 0) %} {% set tt_w = 0.0 if st.get("tts_pct", 0) < 0 else st.get("tts_pct", 0) %} {% set tt_m = _fmt_ms(st.get("tts_ms", 0)) %} {% set tt_p = st.get("tts_pct", 0) %} {% set ov_w = 0.0 if st.get("overhead_pct", 0) < 0 else st.get("overhead_pct", 0) %} {% set ov_m = _fmt_ms(st.get("overhead_ms", 0)) %} {% set ov_p = st.get("overhead_pct", 0) %}
Mean PL {{ avg_s }} (unmasked {{ unm_s }})
p50 {{ p50_s }} (<1s: {{ "%.0f" | format(u1_pct) }}%)
p90 / Max {{ p90_s }} / {{ max_s }}
≤2.0s SLO {{ "%.1f" | format(slo_under_2s) }}% (>3s: {{ "%.0f" | format(o3_pct) }}%)
Filler Masked {{ f_cnt }} turns (saved {{ f_sav }})
Pre-Speech Critical Path ({{ avg_s }} Mean on {{ s_lbl }})
Callbacks {{ cb_m }} ({{ "%.0f" | format(cb_p) }}%) Tools / APIs {{ tl_m }} ({{ "%.0f" | format(tl_p) }}%) LLM {{ lm_m }} ({{ "%.0f" | format(lm_p) }}%) TTS {{ tt_m }} ({{ "%.0f" | format(tt_p) }}%) Gap / Overhead {{ ov_m }} ({{ "%.0f" | format(ov_p) }}%)
{% endif %} {% endfor %}

🔧 Table A: Pre-Speech Component Breakdown & Signals (Critical Path)

{% for row in suite_pl.get("optimization_targets", [])[:10] %} {% set type_badge = {"LLM": "sim", "Tool / API": "tool", "Callback": "callback", "Guardrail": "golden"}.get(row.get("type", ""), "neutral") %} {% set p90_c = _pl_status_cls(row.get("p90_ms")) %} {% set share_pct = (row.get("share_of_pl_pct", 0.0) or 0.0) | float %} {% set share_cls = "fail" if share_pct >= 15.0 else ("warn" if share_pct >= 5.0 else "") %} {% set cpt = (row.get("calls_per_turn", 0.0) or 0.0) | float %} {% set r_type = row.get("type", "") %} {% set r_agent = row.get("agent", "") %} {% set r_imp = _fmt_ms(row.get("avg_pl_impact_ms")) %} {% set r_obs = row.get("observation", row.get("recommendation", "")) or "" %} {% set sub_cbs = row.get("sub_callbacks") or [] %} {% endfor %}
Component / StageCalls (Calls/Turn)Meanp50p90Share of Suite PLObservation
{{ row.get('component', '') }} {{ r_type }}
Agent: {{ r_agent }} {% if sub_cbs %}
▶{% if sub_cbs | length == 1 %}1 callback{% else %}{{ sub_cbs | length }} callbacks in order{% endif %}
{% for sc in sub_cbs %} {% set sc_seq = sc.get("seq", 1) %} {% set sc_nm = sc.get("cb_name", "callback") %} {% set sc_avg = _fmt_ms(sc.get("avg_ms", 0)) %} {% set sc_p90 = _fmt_ms(sc.get("p90_ms", 0)) %} {% set sc_cnt = sc.get("calls", 0) %} {% set sc_cts = sc.get("child_tools") or [] %} {% set sc_has_det = (sc.get("has_detailed_latency") or (sc.get("avg_code_ms", 0) or 0) > 0 or (sc.get("avg_sandbox_ms", 0) or 0) > 0) %} {% set tot_r = (sc.get("avg_ms", 0) | float) | round | int %} {% set code_r = (sc.get("avg_code_ms", 0) | float) | round | int %} {% set wait_r = (sc.get("avg_ext_wait_ms", 0) | float) | round | int %} {% set sbx_r = 0 if (tot_r - code_r - wait_r) < 0 else (tot_r - code_r - wait_r) %}
#{{ sc_seq }} {{ sc_nm }} — avg {{ sc_avg }}{% if sc_has_det %} (code {{ code_r }}ms{% if wait_r > 0 %} + ext_wait {{ wait_r }}ms{% endif %} + sandbox {{ sbx_r }}ms){% endif %}, p90 {{ sc_p90 }} ({{ sc_cnt }} calls){% if sc_cts %} · → {% for ct in sc_cts[:3] %}{{ ct['name'] }} ({{ (ct['avg_ms'] | float) | round | int }}ms){% if not loop.last %}, {% endif %}{% endfor %}{% if sc_cts | length > 3 %}, +{{ sc_cts | length - 3 }} more{% endif %}{% endif %}
{% endfor %}
{% endif %} 		{{ row.get('calls', 0) }} ({{ "%.2g" | format(cpt) }}×/turn) {{ _fmt_ms(row.get('avg_ms')) }} {{ _fmt_ms(row.get('p50_ms')) }} {{ _fmt_ms(row.get('p90_ms')) }} {{ r_obs }}
{% set cb_decomp = suite_pl.get("callback_decomposition") %} {% if cb_decomp and cb_decomp.get("total_calls", 0) > 0 %} {% set tot_turn_s = _fmt_ms(cb_decomp.get("avg_total_per_turn_ms", 0)) %} {% set cpt_val = (cb_decomp.get("calls_per_turn", 0.0) or 0.0) | float %} {% set tot_calls_val = (cb_decomp.get("total_calls", 0) or 0) | int %} {% set c_ms = _fmt_ms(cb_decomp.get("avg_code_per_turn_ms", 0)) %} {% set c_pct = (cb_decomp.get("code_pct", 0.0) or 0.0) | float %} {% set w_ms = _fmt_ms(cb_decomp.get("avg_ext_wait_per_turn_ms", 0)) %} {% set w_pct = (cb_decomp.get("ext_wait_pct", 0.0) or 0.0) | float %} {% set s_ms = _fmt_ms(cb_decomp.get("avg_sandbox_per_turn_ms", 0)) %} {% set s_pct = (cb_decomp.get("sandbox_pct", 0.0) or 0.0) | float %} {% set s_per_call = _fmt_ms(cb_decomp.get("avg_sandbox_per_call_ms", 0)) %}
📦 Callback Time Decomposition — Why callback span duration exceeds pure Python code time Avg {{ tot_turn_s }}/turn across {{ "%.2g" | format(cpt_val) }} callbacks/turn ({{ "{:,}".format(tot_calls_val) }} total calls)
{% if w_pct > 0 %}
{% endif %}
code {{ c_ms }}/turn ({{ "%.0f" | format(c_pct) }}%): Pure Python execution inside the callback sandbox (total code execution latency - waiting for external calls).
ext_wait {{ w_ms }}/turn ({{ "%.0f" | format(w_pct) }}%): Time paused inside the callback awaiting synchronous tool / OpenAPI calls (waiting for external calls).
sandbox {{ s_ms }}/turn ({{ "%.0f" | format(s_pct) }}%, ~{{ s_per_call }}/call): Per-callback sandbox IPC, session.state serialization/diffing & sandbox init (span_duration - code - ext_wait).
{% endif %} {% if suite_pl.get("filler_table") %}

🗣️ Table B: Pre-Speech Tool Masking & Dead-Air Signals

{% for frow in suite_pl.get("filler_table", []) %} {% set s_cls = frow.get("status_cls", "neutral") %} {% set default_badge = frow.get('status', '') ~ " (" ~ frow.get('masked_calls', 0) ~ "/" ~ frow.get('calls', 0) ~ " Masked)" %} {% set badge_lbl = frow.get("badge_label", default_badge) %} {% set f_agent = frow.get("agent", "") %} {% set f_avg = _fmt_ms(frow.get("avg_ms")) %} {% set f_p90 = _fmt_ms(frow.get("p90_ms")) %} {% set f_obs = frow.get("observation", frow.get("action", "")) or "" %} {% endfor %}
Operation / ToolMean / p90 DurUnmasked RatePost-Filler Dead AirObservation
{{ frow.get('tool_name', '') }} ({{ f_agent }}) {{ f_avg }} / {{ f_p90 }} {{ badge_lbl }} {% if frow.get("masked_calls", 0) > 0 and frow.get("unmasked_calls", 0) == 0 %} 0 ms (Masked · Saved {{ _fmt_ms(frow.get("net_saved_ms")) }}) {% elif frow.get("pl_unmasked_ms") is not none %} {% set u_cls = _pl_status_cls(frow.get("pl_unmasked_ms")) %} {{ _fmt_ms(frow.get("pl_unmasked_ms")) }} PL (No filler) {% else %} 0 ms (Post-speech) {% endif %} {{ f_obs }}
{% endif %}
{% endif %} {% endmacro %} {% macro render_audio_bar(key, label, val_str, pct, color, onclick="", extra_text="", extra_color="var(--text-muted)", extra_bold=false, tip="") %} {% set width = [1.0, [100.0, pct | float] | min] | max %} {% endmacro %} {% macro render_suite_audio_quality(suite_audio) %} {% if suite_audio %} {% set tips = suite_audio.tooltips or {} %}

Audio & Conversation Quality

Naturalness grading (1–5 scale), language & language-switch adherence, multimodal audio checks, and spoken caller-experience issues.

Grade distribution ?

{{ suite_audio.gpa_meta }} ?

{% set any_graded = false %} {% for b in suite_audio.grade_bands %} {% if b.count > 0 %} {% set any_graded = true %}
{% endif %} {% endfor %} {% if not any_graded %}
{% endif %}
{% for b in suite_audio.grade_bands %} {% endfor %}

Audio checks & flow ?

Multimodal audio analyses & conversation checks

{% set any_chk = false %} {% if suite_audio.acoustic_summary and (suite_audio.acoustic_summary.measured_runs or 0) > 0 %} {% set ac = suite_audio.acoustic_summary %} {% set any_chk = true %} {% set ac_col = "var(--good)" if (ac.consistency_pct or 0) >= 90 else ("var(--warn)" if (ac.consistency_pct or 0) >= 70 else "var(--critical)") %} {% set ac_extra_txt = (ac.clean_runs or 0) ~ "/" ~ (ac.measured_runs or 0) ~ " clean" %} {{ render_audio_bar("acoustic_consistency", "Acoustic Signal Consistency", (ac.consistency_pct or 0) ~ "%", ac.consistency_pct or 0, ac_col, "filterAudioRunsByIssue('tts_cache_seam')", extra_text=ac_extra_txt, tip=tips.acoustic_consistency or "") }} {% endif %} {% for chk in suite_audio.audio_checks %} {% if chk.total_count > 0 %} {% set any_chk = true %} {{ render_audio_bar(chk.key, chk.label, chk.pass_count ~ "/" ~ chk.total_count ~ " passed (" ~ chk.pct ~ "%)", chk.pct, chk.color, "filterAudioRunsByIssue('" ~ chk.key ~ "')", tip=chk.tip) }} {% endif %} {% endfor %} {% for cf in suite_audio.conv_factors %} {% if cf.count > 0 and cf.mean_score is not none %} {% set any_chk = true %} {{ render_audio_bar(cf.key, cf.label, "%.2f" | format(cf.mean_score) ~ " / 5.0", cf.pct, cf.color, "filterAudioRunsByIssue('" ~ cf.key ~ "')", tip=cf.tip) }} {% endif %} {% endfor %} {% if not any_chk %}
No multimodal checks or conversation factors recorded
{% endif %}

Language & language switch ?

Routed language code vs agent spoken language

{% set any_lang = false %} {% for lb in suite_audio.language_bars %} {% set any_lang = true %} {% set lb_extra_txt = (lb.mismatch_count ~ " mismatch") if lb.mismatch_count > 0 else "" %} {{ render_audio_bar(lb.key, lb.label, lb.count ~ " (" ~ lb.pct ~ "%)", lb.pct, lb.color, "filterAudioRunsByLang('" ~ lb.key ~ "')", extra_text=lb_extra_txt, extra_color="var(--critical)", extra_bold=true, tip=lb.tip) }} {% endfor %} {% if not any_lang %}
No language data available
{% endif %}

Most common audio & caller-experience issues ?

Flagged across naturalness factors, language switch, audio checks & latency — click any bar to filter runs

{% set any_issues = false %} {% for ib in suite_audio.issue_bars %} {% set any_issues = true %} {{ render_audio_bar(ib.key, ib.label, ib.count ~ " (" ~ ib.pct ~ "%)", ib.pct, ib.color, "filterAudioRunsByIssue('" ~ ib.key ~ "')", tip=ib.tip) }} {% endfor %} {% if not any_issues %}
✓ No audio or caller-experience issues detected across graded runs
{% endif %}

Naturalness Factors & Run Diagnostics

Turn-level vocal & linguistic factor averages (1–5 scale), multimodal audio check breakdown, and per-run explorer.

Turn-level naturalness factors ?

Mean score across all graded agent turns (1.0 = bot-like, 5.0 = human-like)

{% set any_tf = false %} {% for tf in suite_audio.turn_factors %} {% if tf.count > 0 and tf.mean_score is not none %} {% set any_tf = true %} {% set tf_extra_txt = (tf.low_count ~ " low (≤2)") if tf.low_count > 0 else "" %} {{ render_audio_bar(tf.key, tf.label, "%.2f" | format(tf.mean_score) ~ " / 5.0", tf.pct, tf.color, "filterAudioRunsByIssue('" ~ tf.key ~ "')", extra_text=tf_extra_txt, extra_color="var(--critical)", tip=tf.tip) }} {% endif %} {% endfor %} {% if not any_tf %}
No turn-level factor data available
{% endif %}

Conversation-level factors & multimodal checks ?

Whole-call coherence, voice consistency, pause & cutoff checks

{% set any_cf = false %} {% if suite_audio.acoustic_summary and (suite_audio.acoustic_summary.measured_runs or 0) > 0 %} {% set ac = suite_audio.acoustic_summary %} {% set any_cf = true %} {% set ac_col = "var(--good)" if (ac.consistency_pct or 0) >= 90 else ("var(--warn)" if (ac.consistency_pct or 0) >= 70 else "var(--critical)") %} {% set ac_extra_txt = (ac.clean_runs or 0) ~ "/" ~ (ac.measured_runs or 0) ~ " clean" %} {{ render_audio_bar("acoustic_consistency", "Acoustic Signal Consistency & Drift", (ac.consistency_pct or 0) ~ "%", ac.consistency_pct or 0, ac_col, "filterAudioRunsByIssue('tts_cache_seam')", extra_text=ac_extra_txt, tip=tips.acoustic_consistency or "") }} {% endif %} {% for cf in suite_audio.conv_factors %} {% if cf.count > 0 and cf.mean_score is not none %} {% set any_cf = true %} {{ render_audio_bar(cf.key, cf.label, "%.2f" | format(cf.mean_score) ~ " / 5.0", cf.pct, cf.color, "filterAudioRunsByIssue('" ~ cf.key ~ "')", tip=cf.tip) }} {% endif %} {% endfor %} {% for chk in suite_audio.audio_checks %} {% if chk.total_count > 0 %} {% set any_cf = true %} {{ render_audio_bar(chk.key, chk.label, chk.pass_count ~ "/" ~ chk.total_count ~ " passed (" ~ chk.pct ~ "%)", chk.pct, chk.color, "filterAudioRunsByIssue('" ~ chk.key ~ "')", tip=chk.tip) }} {% endif %} {% endfor %} {% set wer_s = suite_audio.wer_summary %} {% if wer_s and wer_s.total_count > 0 and wer_s.mean_wer is not none %} {% set any_cf = true %} {% set m_wer = wer_s.mean_wer %} {% set pct_wer = [0.0, [100.0, (1.0 - m_wer) * 100.0] | min] | max %} {% set col_wer = "var(--good)" if m_wer <= 0.10 else ("var(--warn)" if m_wer <= 0.25 else "var(--critical)") %} {% set wer_extra_txt = wer_s.high_wer_count ~ " high (>25%)" %} {{ render_audio_bar("wer", "Transcription Word Error Rate (WER)", "%.1f%% WER" | format(m_wer * 100.0), pct_wer, col_wer, "filterAudioRunsByIssue('high_wer')", extra_text=wer_extra_txt, tip=tips.wer_tip) }} {% endif %} {% set b_stats = suite_audio.barge_stats %} {% if b_stats and b_stats.barge_turns > 0 %} {% set any_cf = true %} {% set b_rate_pct = [0.0, [100.0, b_stats.barge_in_rate * 100.0] | min] | max %} {% set col_barge = "var(--warn)" if b_stats.barge_in_rate > 0.3 else "var(--good)" %} {% set barge_extra_txt = (b_stats.sessions_with_barge_in ~ " sessions") if b_stats.sessions_with_barge_in else "" %} {{ render_audio_bar("barge_in", "Caller Barge-In & Interruption Rate", b_stats.barge_turns ~ " barge-in turns (" ~ "%.1f%%" | format(b_stats.barge_in_rate * 100.0) ~ ")", b_rate_pct, col_barge, "filterAudioRunsByIssue('verdict_barged_missed')", extra_text=barge_extra_txt, tip="Percentage of conversation turns where the caller interrupted or spoke over agent audio.") }} {% endif %} {% if not any_cf %}
No conversation factors or multimodal check metrics available
{% endif %}
{% endif %} {% endmacro %} {% macro render_suite_conversations(suite_audio) %} {% if suite_audio %} {% set tips = suite_audio.tooltips or {} %} {% set total_runs = suite_audio.total_runs or 0 %} {% set pager_pages = [1, (total_runs + 49) // 50] | max %}

Conversations

Showing {{ total_runs }} of {{ total_runs }} conversations

🔍 /
{% set p_cnt = (suite_audio.run_rows | selectattr('passed') | list | length) if suite_audio.run_rows else 0 %} {% set f_cnt = total_runs - p_cnt %}
Outcome
{% set hl_cnt = suite_audio.grade_bands[0].count if (suite_audio.grade_bands and suite_audio.grade_bands | length > 0) else 0 %} {% set tr_cnt = suite_audio.grade_bands[1].count if (suite_audio.grade_bands and suite_audio.grade_bands | length > 1) else 0 %} {% set bl_cnt = suite_audio.grade_bands[2].count if (suite_audio.grade_bands and suite_audio.grade_bands | length > 2) else 0 %}
Grade
{% set sim_cnt = (suite_audio.run_rows | selectattr('eval_type', 'equalto', 'sim') | list | length) if suite_audio.run_rows else 0 %} {% set gld_cnt = (suite_audio.run_rows | selectattr('eval_type', 'equalto', 'golden') | list | length) if suite_audio.run_rows else 0 %}
Eval Type
Language
{% for lb in suite_audio.language_bars %} {% set val_key = "switch" if lb.key == "language_switch" else lb.key %} {% endfor %} {% set mismatch_cnt = (suite_audio.run_rows | selectattr('language_mismatch') | list | length) if suite_audio.run_rows else 0 %} {% if mismatch_cnt > 0 %} {% endif %}
{% set chk_pass_cnt = (suite_audio.run_rows | selectattr('audio_checks_status', 'equalto', 'PASS') | list | length) if suite_audio.run_rows else 0 %} {% set chk_fail_cnt = (suite_audio.run_rows | selectattr('audio_checks_status', 'equalto', 'FAIL') | list | length) if suite_audio.run_rows else 0 %}
Audio Checks
{% if suite_audio.intents_list %}
Intent
{% for it in suite_audio.intents_list %} {% set it_key = it.key if it is mapping else it %} {% set it_lbl = it.label if it is mapping else it %} {% set it_cnt = it.count if it is mapping else "" %} {% endfor %}
{% endif %}
Issues / Flags any of
{% for ib in suite_audio.issue_bars %} {% endfor %}
{% for r in suite_audio.run_rows %} {% set score_str = "%.1f/5" | format(r.score) if r.score is not none else "—" %} {% set pass_str = "PASS" if r.passed else "FAIL" %} {% set pass_cls = "pass" if r.passed else "fail" %} {% set pl_str = "%.0f ms" | format(r.pl_p50_ms) if r.pl_p50_ms is not none else "—" %} {% set wer_str = "%.1f%%" | format(r.wer * 100) if r.wer is not none else "—" %} {% set chk_str = r.audio_checks_status %} {% set chk_cls = "pass" if chk_str == "PASS" else ("fail" if chk_str == "FAIL" else "meta") %} {% set lang_cls = "fail" if r.language_mismatch else "" %} {% set row_id = r.id or r.row_id %} {% else %} {% endfor %}
Run Conversation / Eval Type Outcome Grade ? Score ? Language ? Audio Checks ? Turns Latency p50 ? WER ? Flags / Issues ?
#{{ r.run_num }} {{ r.eval_name | escape }} {% if r.session_id %}
{{ r.session_id | escape }}
{% endif %} 		{{ r.eval_type }} {{ pass_str }} {{ r.grade_label }} {{ score_str }} {{ r.language_status_label | escape }} {{ chk_str }} {{ r.turns_count or 1 }} {{ pl_str }} {{ wer_str }} {% if r.issue_keys %} {% for ik in r.issue_keys %} {% set full_lbl = r.issue_labels[loop.index0] if (r.issue_labels and r.issue_labels | length > loop.index0) else ik %} {% set fl = r.flag_labels[loop.index0] if (r.flag_labels and r.flag_labels | length > loop.index0) else (r.issue_labels[loop.index0] if (r.issue_labels and r.issue_labels | length > loop.index0) else ik) %} {% set bad_issues = ["language_mismatch", "voice_drift", "long_pauses", "agent_looping", "audio_cutoff", "having_trouble", "audio_exp_failed", "verdict_barged_missed", "subagent_ping_pong_loop", "ignored_user", "markdown_link_tts_leakage", "prompt_or_directive_leakage", "midturn_voice_seam", "audio_truncation"] %} {% set cls_name = "bad" if ik in bad_issues else "warn" %} {% set tip_txt = tips.issue_tips.get(ik, "") if (tips.issue_tips is mapping) else "" %} {{ fl | escape }} {% endfor %} {% else %} None {% endif %}
No evaluation runs recorded
Showing {{ 1 if total_runs else 0 }}–{{ [total_runs, 50] | min }} of {{ total_runs }} 1 / {{ pager_pages }}
{% endif %} {% endmacro %} {% macro render_audio_drawers(suite_audio) %} {% if suite_audio %}
{% endif %} {% endmacro %} {% macro render_suite_audio(suite_audio) %} {% if suite_audio %} {{ render_suite_audio_quality(suite_audio) }} {{ render_suite_conversations(suite_audio) }} {{ render_audio_drawers(suite_audio) }} {% endif %} {% endmacro %} {% macro render_standalone_report_scaffold(overview_html, suite_audio) %}
{{ overview_html | safe }}
{% if suite_audio and suite_audio.has_audio_analysis %}
{{ render_suite_audio_quality(suite_audio) }}
{% endif %} {% if suite_audio and suite_audio.total_runs > 0 %}
{{ render_suite_conversations(suite_audio) }}
{% endif %}
{% if suite_audio %} {{ render_audio_drawers(suite_audio) }} {% endif %} {% endmacro %} {% macro render_ces_audio_player(ces_audio) %} {% if not ces_audio or not ces_audio.has_audio %}
{{ (ces_audio.audio_note if ces_audio else "No server-side recording found for this session.") | escape }}
{% else %} {% set aid = ces_audio.anchor %}
🎧 Full call (L=caller, R=agent):{{ "%.2f" | format(ces_audio.duration_s) }}s
Listen to:
Jump to:{% for ch in ces_audio.chapters %}{% endfor %}
{% endif %} {% endmacro %} {% macro render_ces_turn_vad(turn_vad) %} {% if turn_vad %} {% if turn_vad.turn_no == 1 and turn_vad.setup_s is not none %}⏱ call setup before greeting: {{ "%.2f" | format(turn_vad.setup_s) }}s{% elif turn_vad.turn_no > 1 and turn_vad.dead_air_s is not none %}⏱ dead air {{ "%.2f" | format(turn_vad.dead_air_s) }}s{% endif %}{% if turn_vad.segments %}
{% for seg in turn_vad.segments %}
{{ seg.text | escape }}
{% endfor %}
{% endif %} {% endif %} {% endmacro %} {% macro render_enriched_sim_report(data) %}

{{ data.title | escape }}

{% if data.source_report_uri %}Source report: {{ data.source_report_uri | escape }}  |  {% endif %}Generated {{ data.generated_at | escape }}  |  App: {% if data.ces_base %}{{ data.app_id | escape }} ↗{% else %}{{ data.app_id | escape }}{% endif %}  |  Audio: {{ data.recordings_uri | escape }}/<date>/<session_id>/
{% for kpi in data.kpis %}
{{ kpi.label | escape }}
{{ kpi.value | escape }}
{{ kpi.sub | escape }}
{% endfor %}

Dead air = silence between the end of the caller's utterance (channel 0) and the first agent audio (channel 1) in the server-recorded full-session(k).wav. Turn 1 is reported separately as call setup (bootstrap before greeting). Timelines use a simple RMS VAD ({{ data.vad_dbfs }} dBFS, {{ data.frame_ms }} ms frames).

Failures ({{ data.failures | length }})

Slowest agent responses (dead air)

{% for w in data.worst_rows %}{% endfor %}
#Scenario · turnDead air
{{ w.idx }}{{ w.name | escape }} · run {{ w.run_no }} · turn {{ w.turn_no }}{{ "%.2f" | format(w.dead_air_s) }}s{% if w.seek_at is not none %}{% endif %}

All runs

{% for row in data.toc_rows %}{% endfor %}
#ScenarioTags StatusGoalsExpect.TurnsAvg dead air Max dead airSetup
{{ row.idx }}{{ row.name | escape }}{% for tag in row.tags %}{{ tag.text | escape }}{% endfor %}{{ 'PASS' if row.passed else 'FAIL' }}{{ row.goals | escape }}{{ row.expectations | escape }}{{ row.turns | escape }}{% if row.avg_dead_air_s is not none %}{{ "%.2f" | format(row.avg_dead_air_s) }}s{% else %}—{% endif %}{% if row.max_dead_air_s is not none %}{{ "%.2f" | format(row.max_dead_air_s) }}s{% else %}—{% endif %}{% if row.setup_s is not none %}{{ "%.1f" | format(row.setup_s) }}s{% else %}—{% endif %}
{% for run in data.runs %}

{{ run.idx }}. {{ run.name | escape }} {{ 'PASS' if run.passed else 'FAIL' }}

{% for tag in run.tags %}{{ tag.text | escape }}{% endfor %}
Run {{ run.run_no }} · goals {{ run.goals | escape }} · expectations {{ run.expectations | escape }} · {{ run.turns | escape }} turns · voice {{ run.voice | escape }} · {% if run.session_link %}Session {{ run.session_id | escape }} ↗{% else %}Session {{ (run.session_id or 'n/a') | escape }}{% endif %}{% if run.ces_audio and run.ces_audio.gcs_dir %} · {{ run.ces_audio.gcs_dir | escape }}{% endif %}
{% if run.ces_audio and run.ces_audio.avg_dead_air_s is not none %}⏱ avg dead air {{ "%.2f" | format(run.ces_audio.avg_dead_air_s) }}s{% endif %}{% if run.ces_audio and run.ces_audio.max_dead_air_s is not none %}max {{ "%.2f" | format(run.ces_audio.max_dead_air_s) }}s{% endif %}{% if run.ces_audio and run.ces_audio.setup_s is not none %}setup {{ "%.2f" | format(run.ces_audio.setup_s) }}s{% endif %}
{% if run.error is not none %}
Error: {{ run.error | escape }}
{% else %}{{ render_ces_audio_player(run.ces_audio) }}{% for tr in run.turn_rows %}{% endfor %}
CallerAgent (server-recorded full-session(k).wav — exact dead air)
Turn {{ tr.turn_no }}
{% if tr.seek_offset_s is not none %} {% endif %}
🗣 "{{ tr.user | escape }}"
🤖 {% if tr.agent_lines %}{% for line in tr.agent_lines %}{{ line | escape }}{% if not loop.last %}
{% endif %}{% endfor %}{% else %}(no agent text){% endif %}
{{ render_ces_turn_vad(tr.turn_vad) }}{% for ev in tr.events %}{% if ev.kind == "tool" %}
🔧 {{ ev.text | escape }}
{% if ev.tool_input %}Input
{{ ev.tool_input | escape }}
{% endif %}{% if ev.tool_output %}Output
{{ ev.tool_output | escape }}
{% endif %}
{% elif ev.kind == "transfer" %}
🔀 {{ ev.text | escape }}
{% else %}
{{ ev.text | escape }}
{% endif %}{% endfor %}
Goals & expectations — goals {{ run.goals | escape }}, expectations {{ run.expectations | escape }}{% for s in run.step_details %}
Goal: {{ s.goal | escape }}
Criteria: {{ s.success_criteria | escape }}
Status: {{ s.status | escape }}
{{ s.justification | escape }}
{% endfor %}{% for e in run.expectation_details %}
{{ e.status | escape }} {% if e.expectation.startswith('[tool] ') %}[tool] {{ e.expectation[7:] | escape }}{% else %}{{ e.expectation | escape }}{% endif %}
{{ e.justification | escape }}
{% endfor %}
Session parameters
{{ run.session_parameters_json | escape }}
{% endif %}
{% endfor %} {% endmacro %} {% macro render_eval_index_dashboard(data) %}

{{ data.title | escape }}

Rolling evaluation runs in Google Cloud Storage
{% for card in data.cards %}
{{ card.suite | escape }} {{ 'PASS' if card.gate_passed else 'FAIL' }}
{{ card.passed_cases }}/{{ card.total_cases }} cases ({{ card.pass_rate_pct_str }})
{{ card.timestamp_utc | escape }} · Latest Report{% if card.latest_enriched_url %} · Latest Enriched{% endif %}
{% endfor %}
{% for row in data.rows %} {% endfor %}
Timestamp (UTC) Suite Gate Cases Passed Changelist / SHA Run ID Artifact
{{ row.timestamp_utc | escape }} {{ row.suite | escape }} {{ 'PASS' if row.gate_passed else 'FAIL' }} {{ row.passed_cases }}/{{ row.total_cases }} ({{ row.pass_rate_pct_str }}) {{ row.git_sha_short | escape }} {{ row.run_id | escape }} Open Report{% if row.enriched_report_url %} · Enriched (Audio+VAD){% endif %}
{% endmacro %}
Eval Report Generated {{ ts }}
/
{% if suite_audio %}
{% endif %}
{{ "%.0f" | format(pct) }}%
{{ passed }}/{{ total }} total passed
Generated {{ ts }}
{% if golden_results %} {% set g_cls = "pass" if g_passed == g_total else "fail" %}
Goldens ({{ golden_modality }})
{{ g_passed }}/{{ g_total }}
{% if g_duration > 0 %}
{{ _fmt_duration(g_duration) }} {% endif %}
{% endif %} {% if sim_results %} {% set s_cls = "pass" if s_passed == s_total else "fail" %} {% set s_time = "" %}
Sims ({{ sim_modality }})
{{ s_passed }}/{{ s_total }}
{% if sim_wall_clock_s %}
{{ _fmt_duration(sim_wall_clock_s) }} {% elif s_dur > 0 %}
~{{ _fmt_duration(s_dur) }} {% endif %}
{% endif %} {% if tool_results %} {% set t_cls = "pass" if t_passed == t_total else "fail" %}
Tool Tests
{{ t_passed }}/{{ t_total }}
{% endif %} {% if callback_results %} {% set c_cls = "pass" if c_passed == c_total else "fail" %}
Callback Tests
{{ c_passed }}/{{ c_total }}
{% endif %}
{% if suite_audio or suite_pl %}
{% if suite_audio and suite_audio.has_audio_analysis and suite_audio.graded_runs > 0 and suite_audio.mean_score is not none %} {% set nat_cls = "pass" if suite_audio.mean_score >= 4.0 else ("warn" if suite_audio.mean_score >= 2.5 else "fail") %}
Audio Naturalness?
{{ "%.2f" | format(suite_audio.mean_score) }}/5
GPA {{ "%.2f" | format(suite_audio.gpa_4) }}/4.0 · {{ suite_audio.graded_runs }}/{{ suite_audio.total_runs }} graded
{% endif %} {% if suite_audio and suite_audio.has_audio_analysis and suite_audio.graded_runs > 0 %} {% set hb = (suite_audio.grade_bands | selectattr('key', 'equalto', 'human_like') | first) if suite_audio.grade_bands else none %} {% if hb %} {% set hb_cls = "pass" if hb.pct >= 75 else ("warn" if hb.pct >= 50 else "fail") %}
Human-like Runs?
{{ hb.pct }}%
{{ hb.count }} / {{ suite_audio.graded_runs }} graded runs (≥4.0)
{% endif %} {% endif %} {% if suite_pl and suite_pl.get("primary") and suite_pl["primary"].get("p50_ms") is not none %} {% set pl_p50 = suite_pl["primary"].get("p50_ms") %}
Perceived Latency (p50)?
{{ _fmt_ms(pl_p50) }}
p90: {{ _fmt_ms(suite_pl["primary"].get("p90_ms")) }} · Mean: {{ _fmt_ms(suite_pl["primary"].get("avg_ms")) }}
{% endif %} {% if suite_pl and suite_pl.get("primary") %} {% set slo_pct = (100.0 - (suite_pl["primary"].get("over_2s_pct", 0.0) | float)) | round(1) %} {% set slo_cls = "pass" if slo_pct >= 80 else ("warn" if slo_pct >= 50 else "fail") %}
Latency ≤2.0s SLO?
{{ "%.1f" | format(slo_pct) }}%
<1s: {{ "%.0f" | format(suite_pl["primary"].get("under_1s_pct", 0.0) | float) }}% · {{ suite_pl.get("total_turns", 0) }} turns
{% endif %} {% if suite_audio and suite_audio.total_runs > 0 and suite_audio.overview_metrics %} {% set om = suite_audio.overview_metrics %}
Avg # LLM Turns?
{{ "%.1f" | format(om.avg_llm_turns) }}
{{ om.total_llm_turns }} total · {{ "%.1f" | format(om.avg_conv_turns) }} user turns/run
{% elif suite_pl and suite_pl.get("avg_llm_turns") is not none %}
Avg # LLM Turns?
{{ "%.1f" | format(suite_pl.get("avg_llm_turns", 0.0)) }}
{{ suite_pl.get("total_llm_turns", 0) }} total · {{ suite_pl.get("total_turns", 0) }} user turns
{% endif %} {% if suite_audio and suite_audio.has_audio_analysis and suite_audio.checks_total_runs %} {% set chk_cls = "pass" if suite_audio.checks_pass_pct >= 90 else ("warn" if suite_audio.checks_pass_pct >= 70 else "fail") %}
Audio Checks Pass Rate?
{{ suite_audio.checks_pass_pct }}%
{{ suite_audio.checks_pass_runs }} / {{ suite_audio.checks_total_runs }} runs passed
{% endif %} {% if suite_audio and suite_audio.has_audio_analysis and suite_audio.acoustic_summary and (suite_audio.acoustic_summary.measured_runs or 0) > 0 %} {% set ac = suite_audio.acoustic_summary %} {% set ac_cls = "pass" if (ac.consistency_pct or 0) >= 90 else ("warn" if (ac.consistency_pct or 0) >= 70 else "fail") %}
Acoustic Consistency?
{{ ac.consistency_pct }}%
{{ ac.clean_runs }} / {{ ac.measured_runs }} drift-free
{% endif %} {% if suite_audio and suite_audio.has_audio_analysis and suite_audio.acoustic_summary and (suite_audio.acoustic_summary.join_runs or 0) > 0 %} {% set ac = suite_audio.acoustic_summary %} {% set jump_cls = "pass" if (ac.jump_free_pct or 0) >= 90 else ("warn" if (ac.jump_free_pct or 0) >= 70 else "fail") %}
Voice Jumps?
{{ ac.jump_free_pct }}%
{{ ac.join_runs - ac.jump_runs }} / {{ ac.join_runs }} runs jump-free · {{ ac.jump_turns }} jump turn{{ "" if ac.jump_turns == 1 else "s" }}
{% endif %} {% if suite_audio and suite_audio.has_audio_analysis and suite_audio.wer_summary and suite_audio.wer_summary.total_count > 0 and suite_audio.wer_summary.mean_wer is not none %} {% set ws = suite_audio.wer_summary %} {% set wer_cls = "pass" if ws.mean_wer <= 0.10 else ("warn" if ws.mean_wer <= 0.25 else "fail") %}
Transcription WER?
{{ "%.1f" | format(ws.mean_wer * 100.0) }}%
{{ ws.high_wer_count }} high (>25%) · {{ ws.total_count }} runs
{% endif %} {% if suite_audio and suite_audio.has_audio_analysis and suite_audio.total_runs > 0 and suite_audio.lang_adherence_pct is not none %} {% set lang_cls = "pass" if suite_audio.lang_mismatch_runs == 0 else ("warn" if suite_audio.lang_adherence_pct >= 85 else "fail") %}
Language Adherence?
{{ suite_audio.lang_adherence_pct }}%
{{ suite_audio.lang_matched_runs }} / {{ suite_audio.total_runs }} matched ({{ suite_audio.lang_mismatch_runs }} mismatch)
{% endif %} {% if suite_audio and suite_audio.has_audio_analysis and suite_audio.barge_stats and suite_audio.barge_stats.barge_turns > 0 %} {% set bs = suite_audio.barge_stats %} {% set barge_cls = "warn" if bs.barge_in_rate > 0.30 else "pass" %}
Caller Barge-In Rate?
{{ "%.1f" | format(bs.barge_in_rate * 100.0) }}%
{{ bs.barge_turns }} turns · {{ bs.sessions_with_barge_in }} sessions
{% endif %}
{% endif %}

All Evals

{% if suite_latency_html or (suite_pl and suite_pl.get("primary")) %} {% endif %}
{% for u in unified %} {% set type_cls = u["type"] %} {% set safe_name = u["name"] | replace("'", "\\'") %} {% set passed_str = "true" if u["passed"] else "false" %} {% endfor %}
ResultTypeEvalDetail
{% if u["type"] == "sim" and "run_results" in u %} {% set p = u["run_results"] | selectattr('passed') | list | length %} {% set t = u["run_results"] | length %} {% if p == t %} PASSED {% elif p == 0 %} FAILED {% else %} MIXED {{ p }}/{{ t }} {% endif %} {% elif u["passed"] %} PASSED {% else %} FAILED {% endif %} {{ u["type"] }} {{ u["name"] | escape }} {% if u["type"] == "sim" and "run_results" in u %} {% for r in u["run_results"] %} {% set dot_cls = "p" if r.get("passed") else "f" %} {% endfor %} {% else %} {{ u["detail"] | escape }} {% endif %}
{% if failure_patterns_html %} {{ failure_patterns_html | safe }} {% endif %}
{% if suite_latency_html %}
{{ suite_latency_html | safe }}
{% elif suite_pl and suite_pl.get("primary") %}
{{ render_suite_latency(suite_pl, trend_chart, _fmt_ms, _pl_status_cls) }}
{% endif %} {% if golden_results %}
{% set g_pct_str = "%.0f%%" | format(100 * g_passed / g_total) if g_total else "0%" %}

Goldens ({{ g_passed }}/{{ g_total }} — {{ g_pct_str }})

{% for r in golden_results | sort(attribute='name') | sort(attribute='passed') %} {% set cls = "pass-bg" if r["passed"] else "fail-bg" %} {% set status = "PASS" if r["passed"] else "FAIL" %} {% set failed_str = "false" if r["passed"] else "true" %} {% set passed_str = "true" if r["passed"] else "false" %}
{{ r["name"] | escape }} golden {{ status }}
Golden Evaluation — {{ status }} | {{ r.get('turns',[]) | length }} turns{% if r.get('duration_s') %} | {{ _fmt_duration(r['duration_s']) }}{% endif %} {% if r.get('session_id') %}
Session: {% if ces_base %} {{ r['session_id'] }} {% else %} {{ r['session_id'] }} {% endif %}
{% if golden_modality == "audio" %}
Full Conversation
{% endif %} {% endif %} {% if r.get('session_parameters') %}
⚙ Session Parameters 
{{ json.dumps(r['session_parameters'], indent=2) | escape }}
{% endif %} {% if r.get('_conv_latency_html') %} {{ r['_conv_latency_html'] | safe }} {% endif %}
Conversation & Comparison {% for turn in r.get('turns', []) %} {% set sem = turn.get('semantic_score') %} {% set sem_cls = "sem-" ~ sem if sem is not none else "" %} {% set any_fail = (turn.get('comparisons', []) | selectattr('outcome', 'equalto', 'FAIL') | list | length > 0) or (sem is not none and sem < 3) %} {% set row_cls = "mismatch" if any_fail else "match" %}
Turn {{ turn["index"] }} {% if sem is not none %} {{ sem }}/4 {% if turn.get('semantic_explanation') %} ({{ turn['semantic_explanation'] | escape }}) {% endif %} {% endif %}
{% if turn.get('user_input') %}
User: {{ turn['user_input'] | escape }} {% if golden_modality == "audio" and r.get('session_id') %} {% endif %}
{% elif golden_modality == "audio" and r.get('session_id') %}
{% endif %} {% if golden_modality == "audio" and r.get('session_id') %} {% endif %} {% for comp in turn.get('comparisons', []) %} {% set is_fail = comp["outcome"] == "FAIL" or (ctype == "text" and sem is not none and sem < 3) %} {% set c_cls = "fail-bg" if is_fail else "pass-bg" %} {% set badge_cls = "fail" if is_fail else ("pass" if comp["outcome"] == "PASS" else "") %} {% set ctype = comp.get('type', '?') %} {% set icon = {"text": "💬", "tool_call": "🔧", "transfer": "🔀"}.get(ctype, "") %}
{% if ctype != "text" or comp["outcome"] != "UNSPECIFIED" %} {{ comp["outcome"] }} {% endif %} {{ icon | safe }} {{ ctype }}
Expected: {{ comp.get('expected', '') | string | escape }}
Actual: {{ comp.get('actual', '') | string | escape }} {% if ctype == "tool_call" %} {% if comp.get('tool_invocation_score') is not none %}
Tool Invocation Score: {{ comp['tool_invocation_score'] }}
{% endif %} {% if comp.get('tool_invocation_explanation') %}
Explanation: {{ comp['tool_invocation_explanation'] | escape }}
{% endif %} {% endif %} {% if ctype == "tool_call" and (comp.get('expected_args') or comp.get('actual_args')) %}
Tool call args
{% if comp.get('expected_args') %}
Expected args:
{{ json.dumps(comp['expected_args'], indent=2, default=str) | escape }}
{% endif %} {% if comp.get('actual_args') %}
Actual args:
{{ json.dumps(comp['actual_args'], indent=2, default=str) | escape }}
{% endif %}
{% endif %}
{% endfor %} {% if turn.get('_turn_latency_html') %} {{ turn['_turn_latency_html'] | safe }} {% endif %}
{% endfor %}
{% for exp in r.get('expectations', []) %} {% set e_cls = "met" if exp["status"] == "Met" else "not-met" %}
{{ exp["status"] | escape }} {{ exp["expectation"] | string | truncate(150) | escape }} {% if exp.get('justification') %}
{{ exp['justification'] | string | truncate(250) | escape }} {% endif %}
{% endfor %}
{% endfor %}
{% endif %} {% if sim_results %}
{% set s_pct_str = "%.0f%%" | format(100 * s_passed / s_total) if s_total else "0%" %}

Simulations ({{ s_passed }}/{{ s_total }} — {{ s_pct_str }})

{# We group simulations by name in template or pass them grouped. Let's assume unified has grouped them for summary, but for detail we can loop over sorted results #} {% set grouped_sims = {} %} {% for r in sim_results %} {% if r.name not in grouped_sims %} {% set _ = grouped_sims.update({r.name: {"pass": 0, "total": 0, "runs": []}}) %} {% endif %} {% set _ = grouped_sims[r.name].update({"total": grouped_sims[r.name]["total"] + 1}) %} {% if r.get("passed") %} {% set _ = grouped_sims[r.name].update({"pass": grouped_sims[r.name]["pass"] + 1}) %} {% endif %} {% set _ = grouped_sims[r.name]["runs"].append(r) %} {% endfor %} {% for name, s in grouped_sims.items() | sort(attribute='1.pass', reverse=false) %} {% set score = s['pass'] ~ '/' ~ s['total'] %} {% set cls = "pass-bg" if s['pass'] == s['total'] else "fail-bg" %} {% set passed_str = "true" if s['pass'] == s['total'] else "false" %} {% set first_run = s["runs"][0] if s["runs"] else {} %} {% set first_bg_dict = first_run.get("background_noise") if first_run.get("background_noise") is mapping else {} %} {% set card_bg = first_run.get("background_noise_file") or first_bg_dict.get("preset") or first_bg_dict.get("file") or bg_noise_file %} {% set card_snr = first_run.get("bg_noise_snr") if first_run.get("bg_noise_snr") is not none else (first_bg_dict.get("snr") if first_bg_dict.get("snr") is not none else bg_noise_snr) %} {% set card_level = first_run.get("bg_noise_level") or (snr_to_noise_level(card_snr) if snr_to_noise_level and card_snr is not none else "") %}
{{ name | escape }} sim{% if card_bg %} 🔊 BG: {{ card_bg | escape }}{% if card_snr is not none %} ({% if card_level %}{{ card_level }} · {% endif %}{{ card_snr }} dB SNR){% endif %}{% endif %} {{ score }}
{% for r in s["runs"] %} {% set run_cls = "pass" if r.get("passed") else "fail" %} {% set failed_str = "false" if r.get("passed") else "true" %} {% set r_bg_dict = r.get("background_noise") if r.get("background_noise") is mapping else {} %} {% set run_bg = r.get("background_noise_file") or r_bg_dict.get("preset") or r_bg_dict.get("file") or bg_noise_file %} {% set run_snr = r.get("bg_noise_snr") if r.get("bg_noise_snr") is not none else (r_bg_dict.get("snr") if r_bg_dict.get("snr") is not none else bg_noise_snr) %} {% set run_level = r.get("bg_noise_level") or (snr_to_noise_level(run_snr) if snr_to_noise_level and run_snr is not none else "") %} {% set run_burst = r.get("burst_noise_files") or r_bg_dict.get("burst_files") or burst_noise_files %} {% set run_burst_snr = r.get("burst_noise_snr") if r.get("burst_noise_snr") is not none else r_bg_dict.get("burst_snr") %} {% set run_burst_level = r.get("burst_noise_level") or (snr_to_noise_level(run_burst_snr) if snr_to_noise_level and run_burst_snr is not none else "") %} {% set ces_aud = r.get("_ces_audio") %}
Run {{ r.get('run','?') }} — {% if r.get('passed') %}PASS{% else %}FAIL{% endif %} | goals: {{ r.get('goals','?') }} | expectations: {{ r.get('expectations','?') }} | turns: {{ r.get('turns','?') }}{% if r.get('naturalness') %} | naturalness: {{ r['naturalness'] }}{% endif %}{% if r.get('duration_s') %} | {{ _fmt_duration(r['duration_s']) }}{% endif %}{% if r.get('sim_user_voice') %} | voice: {{ r['sim_user_voice'] | escape }}{% endif %}{% if ces_aud and ces_aud.get('avg_dead_air_s') is not none %} ⏱ avg dead air {{ "%.2f" | format(ces_aud['avg_dead_air_s']) }}s{% endif %}{% if ces_aud and ces_aud.get('max_dead_air_s') is not none %} max {{ "%.2f" | format(ces_aud['max_dead_air_s']) }}s{% endif %}{% if ces_aud and ces_aud.get('setup_s') is not none %} setup {{ "%.2f" | format(ces_aud['setup_s']) }}s{% endif %} {% if run_bg or run_burst %}
🔊 Background Noise Injection: {% if run_bg %}Continuous preset/file {{ run_bg | escape }}{% if run_level %} at {{ run_level }}{% endif %}{% if run_snr is not none %} ({{ run_snr }} dB SNR){% endif %}{% endif %} {% if run_burst %}{% if run_bg %} · {% endif %}Burst noise ({{ run_burst | length }} file(s){% if run_burst_level %} at {{ run_burst_level }}{% endif %}{% if run_burst_snr is not none %} @ {{ run_burst_snr }} dB SNR{% endif %}): {{ run_burst | join(', ') | escape }}{% endif %}
{% endif %} {% if r.get('session_id') %} {% if ces_base %}
Session: {{ r['session_id'] }}{% if ces_aud and ces_aud.get('gcs_dir') %} · {{ ces_aud['gcs_dir'] | escape }}{% endif %}
{% else %}
Session: {{ r['session_id'] }}{% if ces_aud and ces_aud.get('gcs_dir') %} · {{ ces_aud['gcs_dir'] | escape }}{% endif %}
{% endif %} {% if ces_aud %} {{ render_ces_audio_player(ces_aud) }} {% elif sim_modality == "audio" %}
Full Conversation
{% endif %} {% endif %} {% if r.get('session_parameters') %}
⚙ Session Parameters 
{{ json.dumps(r['session_parameters'], indent=2) | escape }}
{% endif %} {% if r.get('_conv_latency_html') %} {{ r['_conv_latency_html'] | safe }} {% endif %} {% if "error" in r %}
Error: {{ r["error"] | escape }}
{% else %} {% for step in r.get('step_details', []) %} {% set s_cls = "pass" if step["status"] == "Completed" else "fail" %}
Goal: {{ step["goal"] | escape }}
Criteria: {{ step["success_criteria"] | escape }}
Status: {{ step["status"] | escape }} {% if step.get('justification') %}
Justification: {{ step["justification"] | escape }} {% endif %}
{% endfor %} {% for exp in r.get('expectation_details', []) %} {% set e_cls = "met" if exp["status"] == "Met" else "not-met" %} {% set is_tool = exp.get('expectation', '').startswith('[tool] ') %}
{{ exp["status"] | escape }} {% if is_tool %}[tool] {{ exp["expectation"][7:] | escape }}{% else %}{{ exp["expectation"] | escape }}{% endif %} {% if exp.get('justification') %}
Justification: {{ exp["justification"] | escape }} {% endif %}
{% endfor %} {% set nat = r.get('naturalness_details') %} {% if nat %} {% set nat_cls = {'Human-like': 'met', 'Bot-like': 'not-met'}.get(nat.get('overall_label'), '') %}
🎤 Naturalness — {{ nat.get('overall_label','') | escape }} {{ nat.get('overall_score','?') }}/5{% if nat.get('pass_threshold') is not none %} (threshold {{ nat['pass_threshold'] }} → {% if nat.get('passed') %}PASS{% else %}FAIL{% endif %}){% endif %} {% for reason in nat.get('failure_reasons', []) %}
FAILED {{ reason | escape }}
{% endfor %} {% if nat.get('summary') %}
{{ nat['summary'] | escape }}
{% endif %} {% if nat.get('factor_averages') %}
Averages: {% for quality, avg in nat['factor_averages'].items() %} {{ quality | escape }} {{ avg }}/5 {% endfor %}
{% endif %} {% if nat.get('conversation_factors') %}
Conversation level
    {% for f in nat['conversation_factors'] %}
  • {{ f.get('quality','') | escape }} {{ f.get('score','') }}/5{% if f.get('value') %} ({{ f['value'] | escape }}){% endif %} {% if f.get('reason') %}
    {{ f['reason'] | escape }}{% endif %}
    • {% endfor %}
{% endif %} {% for t in nat.get('turns', []) %} {% set t_cls = {'Human-like': 'met', 'Bot-like': 'not-met'}.get(t.get('label'), '') %}
Turn {{ t.get('turn_index','?') }} — {{ t.get('label','') | escape }} {{ t.get('score','?') }}/5 {% if t.get('agent_utterance') %}
{{ t['agent_utterance'] | escape }}
{% endif %} {% if t.get('justification') %}
{{ t['justification'] | escape }}
{% endif %}
    {% for f in t.get('factors', []) %}
  • {{ f.get('quality','') | escape }} {{ f.get('score','') }}/5{% if f.get('value') %} ({{ f['value'] | escape }}){% endif %} {% if f.get('reason') %}
    {{ f['reason'] | escape }}{% endif %}
    • {% endfor %}
{% endfor %}
{% endif %} {% if r.get('_processed_trace') %}
Conversation Trace ({{ r.get('turns','?') }} turns)
{% set ns = namespace(user_turn=0, agent_turn=0) %} {% for item in r['_processed_trace'] %} {% set kind = item[0] %} {% if kind == "user" %} {% set ns.user_turn = ns.user_turn + 1 %}
User: {{ item[1] | escape }} {% if ces_aud and ces_aud.get('has_audio') and ns.user_turn in (ces_aud.get('chapter_offsets') or {}) %} {% elif sim_modality == "audio" and r.get('session_id') and not item[1].startswith("event:") and not item[1].startswith("dtmf:") %} {% endif %}
{% elif kind == "agent" %} {% set ns.agent_turn = ns.agent_turn + 1 %}
Agent: {{ item[1] | escape }} {% if ces_aud and ns.agent_turn in (ces_aud.get('turns') or {}) %}
{{ render_ces_turn_vad(ces_aud['turns'][ns.agent_turn]) }}
{% elif sim_modality == "audio" and r.get('session_id') %} {% endif %}
{% elif kind in ("tool_call", "tool_pair") %} {% set lbl, _, args = item[1].partition(" with args ") %} {% set lbl = lbl.replace("Tool Call: ", "").replace("Tool Call (Output): ", "").split("/")[-1] %}
🔧 {{ lbl | escape }} {% if args %}
Input:
{{ args | escape }}
{% endif %} {% if kind == "tool_pair" %} {% set _, _, result = item[2].partition(" with result ") %} {% if result %}
Output:
{{ result | escape }}
{% endif %} {% endif %}
{% elif kind == "tool_resp" %} {% set lbl, _, result = item[1].partition(" with result ") %} {% set lbl = lbl.replace("Tool Response: ", "").split("/")[-1] %}
📤 {{ lbl | escape }} {% if result %} 
{{ result | escape }}
{% endif %}
{% elif kind == "callback_group" %}
▶⏱ Callback Spans ({{ item[1] | length }})
{% for cbl in item[1] %}
{{ cbl | escape }}
{% endfor %}
{% elif kind == "turn_latency" %} {{ item[1] | safe }} {% else %}
{{ item[1] | escape }}
{% endif %} {% endfor %}
{% endif %} {% endif %}
{% endfor %}
{% endfor %}
{% endif %} {% if tool_results or callback_results %}
{% if tool_results_html %} {{ tool_results_html | safe }} {% endif %} {% if callback_results_html %} {{ callback_results_html | safe }} {% endif %}
{% endif %} {% if (suite_audio and suite_audio.has_audio_analysis) or suite_audio_html %}
{% if suite_audio %} {{ render_suite_audio_quality(suite_audio) }} {% elif suite_audio_html %} {{ suite_audio_html | safe }} {% endif %}
{% endif %} {% if suite_audio and suite_audio.total_runs > 0 %}
{{ render_suite_conversations(suite_audio) }}
{% endif %}
{% if suite_audio %} {{ render_audio_drawers(suite_audio) }} {% endif %}