import type { VoiceTestRunResult } from "./runner"; import type { VoiceMetricName } from "./schema"; export type ReportLocale = "zh-CN" | "en"; type ReportCopy = { htmlLang: string; titleSuffix: string; eyebrow: string; timeLabel: string; recommendation: { passed: string; failed: string }; metrics: { scenarios: string; turns: string; assertions: string; failures: string; scenarioUnit: (count: number) => string; turnUnit: (count: number) => string; assertionUnit: (count: number) => string; failureUnit: (count: number) => string; }; status: { passed: string; failed: string }; scenarioTurns: (count: number) => string; turnMeta: (input: { index: number; latencyMs: number; passed: boolean; assertions: number; failures: number }) => string; customer: string; agent: string; businessRiskLabel: string; riskHeading: string; passedHeading: string; conversationSample: string; noRisk: string; riskItemPrefix: string; repairLabel: string; audioReplay: string; voiceMetrics: string; metricLabels: Record; passedSummary: (result: VoiceTestRunResult, passedTurns: number) => string; repairAdvice: (code: string) => string; }; const zhCopy: ReportCopy = { htmlLang: "zh-CN", titleSuffix: "智能语音体检单", eyebrow: "智能语音体检单", timeLabel: "检测时间", recommendation: { passed: "上线建议:可以进入小范围试点", failed: "上线建议:暂缓上线,先修复高风险对话", }, metrics: { scenarios: "场景", turns: "对话", assertions: "断言", failures: "风险", scenarioUnit: (count) => `${count} 个场景`, turnUnit: (count) => `${count} 轮`, assertionUnit: (count) => `${count} 条断言`, failureUnit: (count) => `${count} 个失败`, }, status: { passed: "通过", failed: "有风险" }, scenarioTurns: (count) => `${count} 轮对话`, turnMeta: ({ index, latencyMs, passed, assertions, failures }) => `第 ${index + 1} 轮 · ${latencyMs}ms · ${passed ? `通过 ${assertions} 条断言` : `${failures} 个风险`}`, customer: "客户", agent: "智能语音", businessRiskLabel: "业务风险", riskHeading: "风险项", passedHeading: "通过项", conversationSample: "对话抽样", noRisk: "本次未发现高风险回复。继续用更多真实客户问题做回归,可以逐步扩大试点范围。", riskItemPrefix: "风险项", repairLabel: "建议修复话术", audioReplay: "音频 replay", voiceMetrics: "语音指标", metricLabels: { timeToFirstWordMs: "首字响应", turnLatencyMs: "整轮延迟", asrLatencyMs: "ASR 延迟", ttsLatencyMs: "TTS 延迟", silenceMs: "长沉默", interruptionCount: "打断次数", asrConfidence: "ASR 置信度", }, passedSummary: (result, passedTurns) => `
  • ${result.summary.scenarios} 个场景完成体检。
  • ${passedTurns} 轮对话通过断言。
  • ${result.summary.assertions} 条断言用于检查价格、档期、转人工、留资和安全边界。
  • `, repairAdvice: repairAdviceForZh, }; const enCopy: ReportCopy = { htmlLang: "en", titleSuffix: "Voice Agent Health Report", eyebrow: "Voice Agent Health Report", timeLabel: "Run time", recommendation: { passed: "Launch advice: ready for a small pilot", failed: "Launch advice: pause launch and fix high-risk turns", }, metrics: { scenarios: "Scenarios", turns: "Turns", assertions: "Assertions", failures: "Risks", scenarioUnit: (count) => `${count} scenario${count === 1 ? "" : "s"}`, turnUnit: (count) => `${count} turn${count === 1 ? "" : "s"}`, assertionUnit: (count) => `${count} assertion${count === 1 ? "" : "s"}`, failureUnit: (count) => `${count} failure${count === 1 ? "" : "s"}`, }, status: { passed: "Passed", failed: "At risk" }, scenarioTurns: (count) => `${count} conversation turn${count === 1 ? "" : "s"}`, turnMeta: ({ index, latencyMs, passed, assertions, failures }) => `Turn ${index + 1} · ${latencyMs}ms · ${passed ? `${assertions} assertions passed` : `${failures} risks`}`, customer: "Customer", agent: "Voice agent", businessRiskLabel: "Business risk", riskHeading: "Risks", passedHeading: "Passed checks", conversationSample: "Conversation sample", noRisk: "No high-risk reply found in this run. Keep expanding coverage with more real customer questions before launch.", riskItemPrefix: "Risk", repairLabel: "Suggested fix", audioReplay: "Audio replay", voiceMetrics: "Voice metrics", metricLabels: { timeToFirstWordMs: "Time to first word", turnLatencyMs: "Turn latency", asrLatencyMs: "ASR latency", ttsLatencyMs: "TTS latency", silenceMs: "Silence", interruptionCount: "Interruptions", asrConfidence: "ASR confidence", }, passedSummary: (result, passedTurns) => `
  • ${result.summary.scenarios} scenario${result.summary.scenarios === 1 ? "" : "s"} completed.
  • ${passedTurns} conversation turn${passedTurns === 1 ? "" : "s"} passed.
  • ${result.summary.assertions} assertions checked pricing, availability, handoff, lead capture, and safety boundaries.
  • `, repairAdvice: repairAdviceForEn, }; const voiceMetricOrder: VoiceMetricName[] = [ "timeToFirstWordMs", "turnLatencyMs", "asrLatencyMs", "ttsLatencyMs", "silenceMs", "asrConfidence", "interruptionCount", ]; function copyFor(locale: ReportLocale): ReportCopy { return locale === "en" ? enCopy : zhCopy; } export function renderJsonReport(result: VoiceTestRunResult): string { return `${JSON.stringify(result, null, 2)}\n`; } export function renderMarkdownSummary(result: VoiceTestRunResult): string { const failedTurns = collectFailedTurns(result); const lines = [ "# Voice Agent TestOps", "", `**Suite:** ${result.suiteName}`, `**Status:** ${result.passed ? "passed" : "failed"}`, `**Run:** ${result.startedAt} - ${result.finishedAt}`, "", `Scenarios: ${result.summary.scenarios} · Turns: ${result.summary.turns} · Assertions: ${result.summary.assertions} · Failures: ${result.summary.failures}`, "", ]; if (failedTurns.length === 0) { lines.push("## Result", "", "No failed checks in this run.", ""); return `${lines.join("\n")}\n`; } lines.push("## Failed Checks", ""); for (const { scenario, turn } of failedTurns) { lines.push(`- ${scenario.title} / turn ${turn.index + 1}`); if (scenario.businessRisk) { lines.push(` - Business risk: ${scenario.businessRisk}`); } const audioReplay = renderMarkdownAudioReplay(turn); if (audioReplay) { lines.push(` - ${audioReplay}`); } const voiceMetrics = renderMarkdownVoiceMetrics(turn); if (voiceMetrics) { lines.push(` - ${voiceMetrics}`); } for (const failure of turn.failures) { lines.push(` - \`${failure.code}\` (${failure.severity}): ${failure.message}`); } } return `${lines.join("\n")}\n`; } export function renderJunitReport(result: VoiceTestRunResult): string { const failureCount = result.summary.failures; const lines = [ '', ``, ` `, ]; for (const scenario of result.scenarios) { for (const turn of scenario.turns) { lines.push( ` `, ); for (const failure of turn.failures) { lines.push( ` ${escapeXml( `${failure.severity}: ${failure.message}`, )}`, ); } lines.push(" "); } } lines.push(" ", ""); return `${lines.join("\n")}\n`; } export function renderHtmlReport(result: VoiceTestRunResult, options: { locale?: ReportLocale } = {}): string { const copy = copyFor(options.locale ?? "zh-CN"); const failedTurns = collectFailedTurns(result); const scenarioRows = result.scenarios .map( (scenario) => `

    ${escapeHtml(scenario.title)}

    ${escapeHtml(scenario.id)} · ${copy.scenarioTurns(scenario.turns.length)}

    ${ scenario.businessRisk ? `

    ${copy.businessRiskLabel}: ${escapeHtml( scenario.businessRisk, )}

    ` : "" }
    ${ scenario.passed ? copy.status.passed : copy.status.failed }
    ${scenario.turns .map( (turn) => `
    ${copy.turnMeta({ index: turn.index, latencyMs: turn.latencyMs, passed: turn.passed, assertions: turn.assertions, failures: turn.failures.length, })}
    ${copy.customer}

    ${escapeHtml(turn.user)}

    ${copy.agent}

    ${escapeHtml(turn.assistant)}

    ${renderVoiceEvidence(turn, copy)} ${renderFailures(turn.failures, copy)}
    `, ) .join("")}
    `, ) .join(""); const recommendation = result.passed ? copy.recommendation.passed : copy.recommendation.failed; return ` ${escapeHtml(result.suiteName)} · ${copy.titleSuffix}

    ${copy.eyebrow}

    ${escapeHtml(result.suiteName)}

    ${copy.timeLabel}: ${escapeHtml(result.startedAt)} - ${escapeHtml(result.finishedAt)}

    ${recommendation}
    ${copy.metrics.scenarios}${result.summary.scenarios}${copy.metrics.scenarioUnit(result.summary.scenarios)}
    ${copy.metrics.turns}${result.summary.turns}${copy.metrics.turnUnit(result.summary.turns)}
    ${copy.metrics.assertions}${result.summary.assertions}${copy.metrics.assertionUnit(result.summary.assertions)}
    ${copy.metrics.failures}${result.summary.failures}${copy.metrics.failureUnit(result.summary.failures)}
    ${renderRiskSummary(failedTurns, copy)} ${renderPassSummary(result, copy)}

    ${copy.conversationSample}

    ${scenarioRows}
    `; } function renderVoiceEvidence( turn: VoiceTestRunResult["scenarios"][number]["turns"][number], copy: ReportCopy, ): string { const audioReplay = renderHtmlAudioReplay(turn, copy); const metrics = renderHtmlVoiceMetrics(turn, copy); if (!audioReplay && !metrics) { return ""; } return `
    ${audioReplay}${metrics}
    `; } function renderHtmlAudioReplay( turn: VoiceTestRunResult["scenarios"][number]["turns"][number], copy: ReportCopy, ): string { const url = turn.audio?.url; if (typeof url !== "string" || url.trim().length === 0) { return ""; } const details = [turn.audio?.label, formatDuration(turn.audio?.durationMs)].filter(Boolean).join(" · "); return `
    ${copy.audioReplay}${details ? `${escapeHtml(details)}` : ""}
    `; } function renderHtmlVoiceMetrics( turn: VoiceTestRunResult["scenarios"][number]["turns"][number], copy: ReportCopy, ): string { const metrics = collectVoiceMetricEntries(turn); if (metrics.length === 0) { return ""; } return `
    ${ copy.voiceMetrics }${metrics .map( ([metric, value]) => `${escapeHtml(copy.metricLabels[metric])}${escapeHtml( formatMetricValue(metric, value), )}`, ) .join("")}
    `; } function renderFailures( failures: VoiceTestRunResult["scenarios"][number]["turns"][number]["failures"], copy: ReportCopy, ): string { if (failures.length === 0) { return ""; } return failures .map( (failure) => `
    ${copy.riskItemPrefix}: ${escapeHtml(failure.code)} · ${escapeHtml( failure.severity, )}
    ${escapeHtml(failure.message)}
    ${copy.repairLabel}: ${escapeHtml( copy.repairAdvice(failure.code), )}
    `, ) .join(""); } function collectFailedTurns(result: VoiceTestRunResult): Array<{ scenario: VoiceTestRunResult["scenarios"][number]; turn: VoiceTestRunResult["scenarios"][number]["turns"][number]; }> { return result.scenarios.flatMap((scenario) => scenario.turns .filter((turn) => turn.failures.length > 0) .map((turn) => ({ scenario, turn, })), ); } function runTimeSeconds(result: VoiceTestRunResult): string { const durationMs = Date.parse(result.finishedAt) - Date.parse(result.startedAt); if (!Number.isFinite(durationMs) || durationMs < 0) { return "0.000"; } return (durationMs / 1000).toFixed(3); } function renderMarkdownAudioReplay(turn: VoiceTestRunResult["scenarios"][number]["turns"][number]): string | undefined { const url = turn.audio?.url; return typeof url === "string" && url.trim().length > 0 ? `Audio replay: ${url}` : undefined; } function renderMarkdownVoiceMetrics(turn: VoiceTestRunResult["scenarios"][number]["turns"][number]): string | undefined { const metrics = collectVoiceMetricEntries(turn); if (metrics.length === 0) { return undefined; } return `Voice metrics: ${metrics.map(([metric, value]) => `${metric}=${formatMetricValue(metric, value)}`).join(", ")}`; } function collectVoiceMetricEntries( turn: VoiceTestRunResult["scenarios"][number]["turns"][number], ): Array<[VoiceMetricName, number]> { return voiceMetricOrder.flatMap((metric) => { const value = turn.voiceMetrics?.[metric]; return typeof value === "number" && Number.isFinite(value) ? [[metric, value] as [VoiceMetricName, number]] : []; }); } function formatMetricValue(metric: VoiceMetricName, value: number): string { if (metric === "asrConfidence") { return `${Math.round(value * 100)}%`; } return metric.endsWith("Ms") ? `${value}ms` : String(value); } function formatDuration(durationMs: number | undefined): string | undefined { return typeof durationMs === "number" && Number.isFinite(durationMs) && durationMs >= 0 ? `${(durationMs / 1000).toFixed(1)}s` : undefined; } function renderRiskSummary( failedTurns: Array<{ scenario: VoiceTestRunResult["scenarios"][number]; turn: VoiceTestRunResult["scenarios"][number]["turns"][number]; }>, copy: ReportCopy, ): string { if (failedTurns.length === 0) { return `

    ${copy.riskHeading}

    ${copy.noRisk}

    `; } return `

    ${copy.riskHeading}

    `; } function renderPassSummary(result: VoiceTestRunResult, copy: ReportCopy): string { const passedTurns = result.scenarios.reduce( (count, scenario) => count + scenario.turns.filter((turn) => turn.passed).length, 0, ); return `

    ${copy.passedHeading}

    `; } function repairAdviceForZh(code: string): string { switch (code) { case "forbidden_pattern_matched": return "删除绝对承诺,改成“需要商家确认”“会尽量沟通和引导”。"; case "expected_phrase_missing": return "补充价格、套餐、档期确认等商家资料引用,先回答客户当前问题。"; case "lead_intent_mismatch": return "调整意图识别规则,让询价、档期、转人工分别落到正确分类。"; case "lead_field_missing": return "在高意向对话中主动收集手机号、称呼或期望时间。"; case "semantic_judge_failed": return "按评测理由复盘回复,收紧提示词、业务事实或转人工条件。"; case "latency_exceeded": return "检查模型链路、网络和工具调用,必要时缩短上下文或使用更快模型。"; default: return "复盘该轮对话,补充提示词或测试样例后重新体检。"; } } function repairAdviceForEn(code: string): string { switch (code) { case "forbidden_pattern_matched": return "Remove absolute promises and route uncertain claims to human confirmation."; case "expected_phrase_missing": return "Answer the current customer question with configured package, price, or availability facts."; case "lead_intent_mismatch": return "Tune intent routing so pricing, availability, booking, and handoff land in the right bucket."; case "lead_field_missing": return "Ask for and extract the required lead field, such as phone, name, or preferred time."; case "semantic_judge_failed": return "Review the judge rationale, then tighten prompts, approved facts, or handoff rules."; case "latency_exceeded": return "Check model latency, network hops, and tool calls; shorten context or use a faster model if needed."; default: return "Review this turn, update the prompt or suite, and rerun the health check."; } } function escapeHtml(value: string): string { return value .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll('"', """) .replaceAll("'", "'"); } function escapeXml(value: string): string { return escapeHtml(value); }