#!/usr/bin/env node import { mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { LeadSource } from "../domain/lead"; import { leadSourceSchema } from "../domain/lead"; import { industrySchema, merchantConfigSchema, type Industry } from "../domain/merchant"; import { createHttpAgent } from "./adapters/httpAgent"; import { createLocalReceptionistAgent } from "./adapters/localReceptionist"; import { createOpenClawAgent } from "./adapters/openClawAgent"; import { createSipAgent } from "./adapters/sipAgent"; import { createTranscriptReplayAgent } from "./adapters/transcriptReplay"; import { parseCliArgs } from "./cliArgs"; import { renderCommercialPilotReport, renderPilotReviewTemplate } from "./commercialReport"; import { diffVoiceTestReports, renderMarkdownDiff } from "./diffReport"; import { buildDoctorProbeFromSuite, diagnoseHttpAgentEndpoint, type DoctorCheck } from "./doctor"; import { exampleCatalog, parseExampleLanguage, type ExampleCatalogEntry, type ExampleLanguage } from "./exampleCatalog"; import { initializeVoiceTestOpsProject } from "./initProject"; import { buildVoiceTestSuiteJsonSchema } from "./jsonSchema"; import { resolveReadablePath } from "./packagePaths"; import { renderProofCard } from "./proofCard"; import { renderHtmlReport, renderJsonReport, renderJunitReport, renderMarkdownSummary } from "./report"; import { parseSemanticJudgeAnnotationSet } from "./annotationSet"; import { buildFailureClusters, buildRegressionSuiteDraft, renderFailureClusterMarkdown, } from "./regressionDraft"; import { buildProductionCallSample, parseProductionCallImport, renderProductionCallSamplingMarkdown, renderProductionCallTranscript, type ProductionCallRecord, type ProductionCallSample, } from "./productionCallImport"; import { analyzeRecordingIntake, renderRecordingIntakeMarkdown } from "./recordingIntake"; import { runVoiceTestSuite, type VoiceTestRunResult } from "./runner"; import type { VoiceTestSeverity, VoiceTestSuite } from "./schema"; import { calibrateSemanticJudge, renderSemanticJudgeCalibrationMarkdown, } from "./semanticJudgeCalibration"; import { loadVoiceTestSuite } from "./suiteLoader"; import { analyzeTranscriptIntake, getTranscriptIntakeDefaults, parseTranscriptIntakePreset, renderTranscriptIntakeMarkdown, type TranscriptIntakePreset, } from "./transcriptIntake"; import { buildDraftMerchantFromTranscript, buildVoiceTestSuiteFromTranscript } from "./transcriptSuite"; const severityRank: Record = { minor: 1, major: 2, critical: 3, }; async function main(argv: string[]): Promise { if (argv[0] === "run") { return runSuite(argv.slice(1)); } if (argv[0] === "from-transcript") { return generateSuiteFromTranscript(argv.slice(1)); } if (argv[0] === "init") { return initProject(argv.slice(1)); } if (argv[0] === "list") { return listExamples(argv.slice(1)); } if (argv[0] === "doctor") { return doctor(argv.slice(1)); } if (argv[0] === "compare") { return compareReports(argv.slice(1)); } if (argv[0] === "draft-regressions") { return draftRegressions(argv.slice(1)); } if (argv[0] === "import-calls") { return importProductionCalls(argv.slice(1)); } if (argv[0] === "recording-intake") { return recordingIntake(argv.slice(1)); } if (argv[0] === "transcript-intake") { return transcriptIntake(argv.slice(1)); } if (argv[0] === "transcript-trial") { return transcriptTrial(argv.slice(1)); } if (argv[0] === "pilot-report") { return generatePilotReport(argv.slice(1)); } if (argv[0] === "proof-card") { return generateProofCard(argv.slice(1)); } if (argv[0] === "calibrate-judge") { return calibrateJudge(argv.slice(1)); } if (argv[0] === "schema") { return exportSchema(argv.slice(1)); } if (argv[0] === "validate") { return validateSuite(argv.slice(1)); } if (argv[0] && !argv[0].startsWith("--")) { throw new Error(`Unknown command: ${argv[0]}`); } return runSuite(argv); } async function initProject(argv: string[]): Promise { const result = await initializeVoiceTestOpsProject(argv); for (const filePath of result.files) { console.log(`Created ${filePath}`); } console.log("Next:"); for (const command of result.nextCommands) { console.log(command); } return 0; } async function exportSchema(argv: string[]): Promise { const args = parseSchemaArgs(argv); const content = `${JSON.stringify(buildVoiceTestSuiteJsonSchema(), null, 2)}\n`; if (args.outPath) { await writeReport(args.outPath, content); console.log(`Wrote JSON Schema: ${args.outPath}`); return 0; } process.stdout.write(content); return 0; } function parseSchemaArgs(argv: string[]): { outPath?: string } { const values = parseKeyValueArgs(argv); for (const option of values.keys()) { if (option !== "out") { throw new Error(`Unknown schema option: --${option}`); } } return { outPath: values.get("out") }; } async function doctor(argv: string[]): Promise { const args = parseDoctorArgs(argv); const suite = args.suitePath ? await loadVoiceTestSuite(args.suitePath) : undefined; const result = await diagnoseHttpAgentEndpoint( args.endpoint, suite ? buildDoctorProbeFromSuite(suite) : undefined, ); console.log("Voice Agent TestOps doctor"); if (suite) { console.log("Suite valid: ok"); console.log(`Probe scenario: ${result.probe.scenarioId}`); } for (const check of result.checks) { console.log(formatDoctorCheck(check)); if (check.advice) { console.log(` fix: ${check.advice}`); } if (check.detail) { console.log(` detail: ${check.detail}`); } } if (result.passed) { console.log("Doctor passed"); return 0; } console.error("Doctor failed"); return 1; } type DoctorArgs = { agent: "http"; endpoint: string; suitePath?: string; }; function parseDoctorArgs(argv: string[]): DoctorArgs { const values = parseKeyValueArgs(argv); for (const option of values.keys()) { if (option !== "agent" && option !== "endpoint" && option !== "suite") { throw new Error(`Unknown doctor option: --${option}`); } } const agent = values.get("agent") ?? "http"; if (agent !== "http") { throw new Error("--agent must be http"); } const endpoint = values.get("endpoint"); if (!endpoint) { throw new Error("--endpoint is required"); } return { agent, endpoint, suitePath: values.get("suite") }; } function formatDoctorCheck(check: DoctorCheck): string { return `${check.label}: ${check.status}`; } function listExamples(argv: string[]): number { const args = parseListArgs(argv); const entries = exampleCatalog.filter( (entry) => (!args.language || entry.language === args.language) && (!args.industry || entry.industry === args.industry), ); console.log("Example suites"); console.log("Use these as references, or generate your own mock data with init.\n"); if (entries.length === 0) { console.log("No examples matched the selected filters."); return 0; } for (const [industryLabel, groupEntries] of groupExamplesByIndustry(entries)) { console.log(`${industryLabel}`); for (const entry of groupEntries) { console.log(` - [${entry.language}] ${entry.title}`); console.log(` ${entry.path}`); console.log(` risks: ${entry.risks}`); } } console.log("\nCreate your own mock suite:"); console.log("npx voice-agent-testops init --industry insurance --lang en --name \"EverSure Insurance\""); console.log("npx voice-agent-testops validate --suite voice-testops/suite.json"); console.log("npx voice-agent-testops run --suite voice-testops/suite.json"); return 0; } type ListExamplesArgs = { language?: ExampleLanguage; industry?: ExampleCatalogEntry["industry"]; }; function parseListArgs(argv: string[]): ListExamplesArgs { const values = parseKeyValueArgs(argv); const args: ListExamplesArgs = {}; for (const option of values.keys()) { if (option !== "lang" && option !== "industry") { throw new Error(`Unknown list option: --${option}`); } } const language = values.get("lang"); if (language) { args.language = parseExampleLanguage(language); } const industry = values.get("industry"); if (industry) { const supportedIndustries = [...new Set(exampleCatalog.map((entry) => entry.industry))]; if (!supportedIndustries.includes(industry as ExampleCatalogEntry["industry"])) { throw new Error(`--industry must be one of: ${supportedIndustries.join(", ")}`); } args.industry = industry as ExampleCatalogEntry["industry"]; } return args; } function groupExamplesByIndustry(entries: ExampleCatalogEntry[]): Array<[string, ExampleCatalogEntry[]]> { const groups = new Map(); for (const entry of entries) { const existing = groups.get(entry.industryLabel) ?? []; existing.push(entry); groups.set(entry.industryLabel, existing); } return [...groups.entries()]; } async function validateSuite(argv: string[]): Promise { const args = parseValidateArgs(argv); const suite = await loadVoiceTestSuite(args.suitePath); const turns = suite.scenarios.reduce((count, scenario) => count + scenario.turns.length, 0); const assertions = suite.scenarios.reduce( (count, scenario) => count + scenario.turns.reduce((turnCount, turn) => turnCount + turn.expect.length, 0), 0, ); console.log(`Suite valid: ${suite.name}`); console.log(`Scenarios: ${suite.scenarios.length}`); console.log(`Turns: ${turns}`); console.log(`Assertions: ${assertions}`); return 0; } function parseValidateArgs(argv: string[]): { suitePath: string } { const values = parseKeyValueArgs(argv); const suitePath = values.get("suite"); if (!suitePath) { throw new Error("--suite is required"); } return { suitePath }; } async function runSuite(argv: string[]): Promise { const args = parseCliArgs(argv); const suite = await loadVoiceTestSuite(args.suitePath); const agent = await createAgentFromArgs(args); const result = await runVoiceTestSuite(suite, agent, { onProgress: (event) => { if (event.type === "turn:start") { console.log( `[${event.scenarioIndex + 1}/${suite.scenarios.length}] ${event.scenarioTitle} - turn ${ event.turnIndex + 1 }/${event.turnTotal}: running`, ); return; } console.log( `[${event.scenarioIndex + 1}/${suite.scenarios.length}] ${event.scenarioTitle} - turn ${ event.turnIndex + 1 }/${event.turnTotal}: ${event.passed ? "passed" : "failed"} (${event.latencyMs}ms, ${ event.failures } failures)`, ); }, }); await writeReport(args.jsonPath, renderJsonReport(result)); await writeReport(args.htmlPath, renderHtmlReport(result, { locale: args.reportLocale })); if (args.summaryPath) { await writeReport(args.summaryPath, renderMarkdownSummary(result)); } if (args.junitPath) { await writeReport(args.junitPath, renderJunitReport(result)); } let newFailureCount: number | undefined; let gatedNewFailureCount: number | undefined; if (args.baselinePath && args.diffMarkdownPath) { const baseline = await readVoiceTestReport(args.baselinePath, "Baseline"); const diff = diffVoiceTestReports(baseline, result); newFailureCount = diff.summary.newFailures; gatedNewFailureCount = args.failOnSeverity ? countFailureRecordsAtOrAboveSeverity(diff.newFailures, args.failOnSeverity) : newFailureCount; await writeReport(args.diffMarkdownPath, renderMarkdownDiff(diff)); } const status = result.passed ? "passed" : "failed"; console.log( `${suite.name}: ${status} (${result.summary.failures} failures, ${result.summary.assertions} assertions)`, ); console.log(`JSON report: ${args.jsonPath}`); console.log(`HTML report: ${args.htmlPath}`); if (args.summaryPath) { console.log(`Markdown summary: ${args.summaryPath}`); } if (args.junitPath) { console.log(`JUnit report: ${args.junitPath}`); } if (args.baselinePath && args.diffMarkdownPath) { console.log(`Baseline report: ${args.baselinePath}`); console.log(`Diff summary: ${args.diffMarkdownPath}`); } if (args.failOnNew) { const newFailures = gatedNewFailureCount ?? newFailureCount ?? 0; console.log(formatNewFailureGate(newFailures, args.failOnSeverity)); return newFailures === 0 ? 0 : 1; } if (args.failOnSeverity) { const gatedFailures = countFailuresAtOrAboveSeverity(result, args.failOnSeverity); console.log( `Severity gate: ${gatedFailures === 0 ? "passed" : "failed"} (${gatedFailures} failures at or above ${ args.failOnSeverity })`, ); return gatedFailures === 0 ? 0 : 1; } return result.passed ? 0 : 1; } async function compareReports(argv: string[]): Promise { const args = parseCompareArgs(argv); const baseline = await readVoiceTestReport(args.baselinePath, "Baseline"); const current = await readVoiceTestReport(args.currentPath, "Current"); const diff = diffVoiceTestReports(baseline, current); console.log( `Voice Agent TestOps diff: ${diff.summary.newFailures} new, ${diff.summary.resolvedFailures} resolved, ${diff.summary.unchangedFailures} unchanged`, ); if (args.diffMarkdownPath) { await writeReport(args.diffMarkdownPath, renderMarkdownDiff(diff)); console.log(`Diff summary: ${args.diffMarkdownPath}`); } if (args.failOnNew) { const newFailures = args.failOnSeverity ? countFailureRecordsAtOrAboveSeverity(diff.newFailures, args.failOnSeverity) : diff.summary.newFailures; console.log(formatNewFailureGate(newFailures, args.failOnSeverity)); return newFailures === 0 ? 0 : 1; } return 0; } async function draftRegressions(argv: string[]): Promise { const args = parseDraftRegressionsArgs(argv); const report = await readVoiceTestReport(args.reportPath, "Failed run"); const sourceSuite = await loadVoiceTestSuite(args.suitePath); const clusters = buildFailureClusters(report); const draftSuite = buildRegressionSuiteDraft(sourceSuite, report); await writeReport(args.outPath, `${JSON.stringify(draftSuite, null, 2)}\n`); if (args.clustersPath) { await writeReport(args.clustersPath, renderFailureClusterMarkdown(report, clusters)); } console.log(`Regression draft: ${args.outPath}`); if (args.clustersPath) { console.log(`Failure clusters: ${args.clustersPath}`); } console.log(`Draft scenarios: ${draftSuite.scenarios.length}`); console.log(`Failure clusters: ${clusters.length}`); return 0; } async function importProductionCalls(argv: string[]): Promise { const args = parseImportCallsArgs(argv); const imported = parseProductionCallImport(await readFile(await resolveReadablePath(args.inputPath), "utf8")); const sample = buildProductionCallSample(imported.records, { sampleSize: args.sampleSize, seed: args.seed, riskOnly: args.riskOnly, rejected: imported.rejected, }); const sampleWithTranscriptPaths = addTranscriptPaths(sample, args.transcriptsDir); if (args.transcriptsDir) { for (const call of sampleWithTranscriptPaths.selectedCalls) { await writeReport(call.transcriptPath ?? "", renderProductionCallTranscript(call)); } } await writeReport(args.outPath, `${JSON.stringify(buildProductionCallManifest(args, sampleWithTranscriptPaths), null, 2)}\n`); if (args.summaryPath) { await writeReport(args.summaryPath, renderProductionCallSamplingMarkdown(sampleWithTranscriptPaths)); } console.log(`Production calls: ${args.inputPath}`); console.log(`Production call sample: ${args.outPath}`); if (args.summaryPath) { console.log(`Sampling summary: ${args.summaryPath}`); } if (args.transcriptsDir) { console.log(`Transcript files: ${args.transcriptsDir}`); } console.log(`Selected calls: ${sampleWithTranscriptPaths.selectedCalls.length}/${sampleWithTranscriptPaths.totalCalls}`); if (sampleWithTranscriptPaths.rejectedCalls.length > 0) { console.log(`Rejected records: ${sampleWithTranscriptPaths.rejectedCalls.length}`); } return 0; } async function generatePilotReport(argv: string[]): Promise { const args = parsePilotReportArgs(argv); const report = await readVoiceTestReport(args.reportPath, "Pilot run"); const options = { customerName: args.customerName, period: args.period, }; if (args.commercialPath) { await writeReport(args.commercialPath, renderCommercialPilotReport(report, options)); console.log(`Commercial pilot report: ${args.commercialPath}`); } if (args.recapPath) { await writeReport(args.recapPath, renderPilotReviewTemplate(report, options)); console.log(`Pilot recap template: ${args.recapPath}`); } return 0; } async function generateProofCard(argv: string[]): Promise { const args = parseProofCardArgs(argv); const report = await readVoiceTestReport(args.reportPath, "Proof card"); const markdown = renderProofCard(report, { customerName: args.customerName, period: args.period, proofUrl: args.proofUrl, nextAsk: args.nextAsk, }); if (args.outPath) { await writeReport(args.outPath, markdown); console.log(`Proof card: ${args.outPath}`); return 0; } process.stdout.write(markdown); return 0; } type CompareArgs = { baselinePath: string; currentPath: string; diffMarkdownPath?: string; failOnNew: boolean; failOnSeverity?: VoiceTestSeverity; }; type DraftRegressionsArgs = { reportPath: string; suitePath: string; outPath: string; clustersPath?: string; }; type ImportCallsArgs = { inputPath: string; outPath: string; summaryPath?: string; transcriptsDir?: string; sampleSize: number; seed: string; riskOnly: boolean; }; type RecordingIntakeArgs = { inputPath: string; summaryPath?: string; }; type TranscriptIntakeArgs = { transcriptPath?: string; readFromStdin: boolean; suitePath: string; merchantPath?: string; merchantOutPath?: string; summaryPath: string; merchantName?: string; industry?: Industry; name?: string; scenarioId?: string; scenarioTitle?: string; source: LeadSource; intake?: TranscriptIntakePreset; turnRole: FromTranscriptTurnRole; }; type TranscriptTrialArgs = { transcriptPath?: string; readFromStdin: boolean; outDir: string; merchantPath?: string; merchantName?: string; industry?: Industry; name?: string; scenarioId?: string; scenarioTitle?: string; source: LeadSource; intake?: TranscriptIntakePreset; customerName?: string; period?: string; proofUrl?: string; failOnSeverity?: VoiceTestSeverity; }; type PilotReportArgs = { reportPath: string; commercialPath?: string; recapPath?: string; customerName?: string; period?: string; }; type ProofCardArgs = { reportPath: string; outPath?: string; customerName?: string; period?: string; proofUrl?: string; nextAsk?: string; }; type CalibrateJudgeArgs = { seedPath: string; outPath?: string; jsonPath?: string; maxExamples?: number; failOnDisagreement: boolean; }; type ProductionCallManifest = { generatedAt: string; sourcePath: string; seed: string; sampleSize: number; totalCalls: number; selectedCalls: Array<{ id: string; provider?: string; startedAt?: string; source: string; industry?: string; riskTags: string[]; customerTurns: number; assistantTurns: number; transcriptPath?: string; }>; rejectedCalls: Array<{ index: number; reason: string }>; riskTagCounts: Array<{ tag: string; count: number }>; }; function parseImportCallsArgs(argv: string[]): ImportCallsArgs { const values = new Map(); const flags = new Set(); const knownValues = new Set(["input", "out", "summary", "transcripts", "sample-size", "seed"]); for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (!arg.startsWith("--")) { throw new Error(`Unexpected argument: ${arg}`); } const name = arg.slice(2); if (name === "risk-only") { flags.add(name); continue; } if (!knownValues.has(name)) { throw new Error(`Unknown import-calls option: --${name}`); } const value = argv[index + 1]; if (!value || value.startsWith("--")) { throw new Error(`${arg} requires a value`); } values.set(name, value); index += 1; } const inputPath = values.get("input"); if (!inputPath) { throw new Error("--input is required"); } const outPath = values.get("out"); if (!outPath) { throw new Error("--out is required"); } const sampleSize = Number.parseInt(values.get("sample-size") ?? "20", 10); if (!Number.isInteger(sampleSize) || sampleSize <= 0) { throw new Error("--sample-size must be a positive integer"); } return { inputPath, outPath, summaryPath: values.get("summary"), transcriptsDir: values.get("transcripts"), sampleSize, seed: values.get("seed") ?? "default", riskOnly: flags.has("risk-only"), }; } function addTranscriptPaths(sample: ProductionCallSample, transcriptsDir: string | undefined): ProductionCallSample { if (!transcriptsDir) { return sample; } return { ...sample, selectedCalls: sample.selectedCalls.map((call) => ({ ...call, transcriptPath: path.join(transcriptsDir, `${safeFileName(call.id)}.txt`), })), }; } function buildProductionCallManifest( args: ImportCallsArgs, sample: ProductionCallSample, ): ProductionCallManifest { return { generatedAt: new Date().toISOString(), sourcePath: args.inputPath, seed: sample.seed, sampleSize: sample.sampleSize, totalCalls: sample.totalCalls, selectedCalls: sample.selectedCalls.map(summarizeProductionCall), rejectedCalls: sample.rejectedCalls, riskTagCounts: sample.riskTagCounts, }; } function summarizeProductionCall(call: ProductionCallRecord): ProductionCallManifest["selectedCalls"][number] { return { id: call.id, provider: call.provider, startedAt: call.startedAt, source: call.source, industry: call.industry, riskTags: call.riskTags, customerTurns: call.transcript.filter((message) => message.role === "customer").length, assistantTurns: call.transcript.filter((message) => message.role === "assistant").length, transcriptPath: call.transcriptPath, }; } async function recordingIntake(argv: string[]): Promise { const args = parseRecordingIntakeArgs(argv); const inputPath = await resolveReadablePath(args.inputPath); const report = analyzeRecordingIntake(await readFile(inputPath, "utf8"), { sourcePath: args.inputPath }); const markdown = renderRecordingIntakeMarkdown(report); if (args.summaryPath) { await writeReport(args.summaryPath, markdown); console.log(`Recording intake summary: ${args.summaryPath}`); } else { process.stdout.write(markdown); } console.log(`Recording intake: ${args.inputPath}`); console.log(`Total recordings: ${report.total}`); console.log(`Ready regression candidates: ${report.readyRegressionCandidates.length}`); console.log(`Issues: ${report.errorCount} errors, ${report.warningCount} warnings`); return report.errorCount === 0 ? 0 : 1; } function parseRecordingIntakeArgs(argv: string[]): RecordingIntakeArgs { const values = parseKeyValueArgs(argv); for (const option of values.keys()) { if (option !== "input" && option !== "summary") { throw new Error(`Unknown recording-intake option: --${option}`); } } const inputPath = values.get("input"); if (!inputPath) { throw new Error("--input is required"); } return { inputPath, summaryPath: values.get("summary"), }; } async function transcriptIntake(argv: string[]): Promise { const args = parseTranscriptIntakeArgs(argv); const transcript = args.readFromStdin ? await readFromStdin() : await readFile(await resolveReadablePath(args.transcriptPath ?? ""), "utf8"); const intakeDefaults = args.intake ? getTranscriptIntakeDefaults(args.intake) : undefined; const merchantName = args.merchantName ?? intakeDefaults?.merchantName; const industry = args.industry ?? intakeDefaults?.industry; const merchant = args.merchantPath ? merchantConfigSchema.parse(JSON.parse(await readFile(await resolveReadablePath(args.merchantPath), "utf8"))) : buildDraftMerchantFromTranscript({ transcript, name: merchantName, industry }); const suite = buildVoiceTestSuiteFromTranscript({ transcript, merchant, name: args.name ?? intakeDefaults?.suiteName, scenarioId: args.scenarioId ?? intakeDefaults?.scenarioId, scenarioTitle: args.scenarioTitle ?? intakeDefaults?.scenarioTitle, source: args.source, turnRole: args.turnRole, }); const suiteOutput = args.merchantOutPath ? buildSuiteWithMerchantRef(suite, relativeMerchantRef(args.suitePath, args.merchantOutPath)) : suite; const report = analyzeTranscriptIntake({ transcript, suite, sourcePath: args.readFromStdin ? undefined : args.transcriptPath, selectedTurnRole: args.turnRole, artifacts: { suitePath: args.suitePath, merchantPath: args.merchantOutPath, summaryPath: args.summaryPath, }, }); if (args.merchantOutPath) { await writeReport(args.merchantOutPath, `${JSON.stringify(merchant, null, 2)}\n`); } await writeReport(args.suitePath, `${JSON.stringify(suiteOutput, null, 2)}\n`); await writeReport(args.summaryPath, renderTranscriptIntakeMarkdown(report)); console.log(`Transcript intake summary: ${args.summaryPath}`); console.log(`Generated suite draft: ${args.suitePath}`); if (args.merchantOutPath) { console.log(`${args.merchantPath ? "Merchant profile" : "Merchant draft"}: ${args.merchantOutPath}`); } console.log(`Transcript: ${args.readFromStdin ? "read from stdin" : args.transcriptPath}`); if (args.intake) { console.log(`Transcript intake: ${args.intake}`); } console.log(`Suite: ${suite.name}`); console.log(`Scenario: ${suite.scenarios[0].id} - ${suite.scenarios[0].title}`); printTurnCount(args.turnRole, suite.scenarios[0].turns.length); console.log(`Assertions: ${report.assertionCount}`); console.log(`Risk signals: ${report.riskSignals.length}`); console.log(`Privacy warnings: ${report.privacyWarnings.length}`); return 0; } async function transcriptTrial(argv: string[]): Promise { const args = parseTranscriptTrialArgs(argv); const transcript = args.readFromStdin ? await readFromStdin() : await readFile(await resolveReadablePath(args.transcriptPath ?? ""), "utf8"); const paths = transcriptTrialPaths(args.outDir); const intakeDefaults = args.intake ? getTranscriptIntakeDefaults(args.intake) : undefined; const merchantName = args.merchantName ?? intakeDefaults?.merchantName; const industry = args.industry ?? intakeDefaults?.industry; const merchant = args.merchantPath ? merchantConfigSchema.parse(JSON.parse(await readFile(await resolveReadablePath(args.merchantPath), "utf8"))) : buildDraftMerchantFromTranscript({ transcript, name: merchantName, industry }); const suite = buildVoiceTestSuiteFromTranscript({ transcript, merchant, name: args.name ?? intakeDefaults?.suiteName, scenarioId: args.scenarioId ?? intakeDefaults?.scenarioId, scenarioTitle: args.scenarioTitle ?? intakeDefaults?.scenarioTitle, source: args.source, turnRole: "customer", }); const suiteOutput = buildSuiteWithMerchantRef(suite, relativeMerchantRef(paths.suitePath, paths.merchantPath)); const intakeReport = analyzeTranscriptIntake({ transcript, suite, sourcePath: args.readFromStdin ? undefined : args.transcriptPath, selectedTurnRole: "customer", artifacts: { suitePath: paths.suitePath, merchantPath: paths.merchantPath, summaryPath: paths.intakeSummaryPath, }, }); const result = await runVoiceTestSuite(suite, createTranscriptReplayAgent({ transcript, source: args.source }), { onProgress: (event) => { if (event.type === "turn:start") { console.log( `[${event.scenarioIndex + 1}/${suite.scenarios.length}] ${event.scenarioTitle} - turn ${ event.turnIndex + 1 }/${event.turnTotal}: replaying transcript`, ); return; } console.log( `[${event.scenarioIndex + 1}/${suite.scenarios.length}] ${event.scenarioTitle} - turn ${ event.turnIndex + 1 }/${event.turnTotal}: ${event.passed ? "passed" : "failed"} (${event.latencyMs}ms, ${ event.failures } failures)`, ); }, }); const pilotOptions = { customerName: args.customerName ?? merchant.name, period: args.period ?? "transcript-only trial", }; const proofUrl = args.proofUrl; await writeReport(paths.merchantPath, `${JSON.stringify(merchant, null, 2)}\n`); await writeReport(paths.suitePath, `${JSON.stringify(suiteOutput, null, 2)}\n`); await writeReport(paths.intakeSummaryPath, renderTranscriptIntakeMarkdown(intakeReport)); await writeReport(paths.reportJsonPath, renderJsonReport(result)); await writeReport(paths.reportHtmlPath, renderHtmlReport(result)); await writeReport(paths.runSummaryPath, renderMarkdownSummary(result)); await writeReport(paths.junitPath, renderJunitReport(result)); await writeReport(paths.commercialPath, renderCommercialPilotReport(result, pilotOptions)); await writeReport(paths.recapPath, renderPilotReviewTemplate(result, pilotOptions)); await writeReport( paths.proofCardPath, renderProofCard(result, { ...pilotOptions, proofUrl, }), ); if (result.summary.failures > 0) { const clusters = buildFailureClusters(result); const draftSuite = buildRegressionSuiteDraft(suite, result); await writeReport(paths.regressionDraftPath, `${JSON.stringify(draftSuite, null, 2)}\n`); await writeReport(paths.failureClustersPath, renderFailureClusterMarkdown(result, clusters)); } console.log(`Transcript trial: ${args.outDir}`); console.log(`Generated suite: ${paths.suitePath}`); console.log(`Merchant draft: ${paths.merchantPath}`); console.log(`Intake summary: ${paths.intakeSummaryPath}`); console.log(`JSON report: ${paths.reportJsonPath}`); console.log(`HTML report: ${paths.reportHtmlPath}`); console.log(`Markdown summary: ${paths.runSummaryPath}`); console.log(`Commercial pilot report: ${paths.commercialPath}`); console.log(`Pilot recap template: ${paths.recapPath}`); console.log(`Proof card: ${paths.proofCardPath}`); if (result.summary.failures > 0) { console.log(`Regression draft: ${paths.regressionDraftPath}`); console.log(`Failure clusters: ${paths.failureClustersPath}`); } console.log(`${suite.name}: ${result.passed ? "passed" : "failed"} (${result.summary.failures} failures, ${result.summary.assertions} assertions)`); if (args.failOnSeverity) { const gatedFailures = countFailuresAtOrAboveSeverity(result, args.failOnSeverity); console.log( `Severity gate: ${gatedFailures === 0 ? "passed" : "failed"} (${gatedFailures} failures at or above ${ args.failOnSeverity })`, ); return gatedFailures === 0 ? 0 : 1; } return 0; } function transcriptTrialPaths(outDir: string): { suitePath: string; merchantPath: string; intakeSummaryPath: string; reportJsonPath: string; reportHtmlPath: string; runSummaryPath: string; junitPath: string; commercialPath: string; recapPath: string; proofCardPath: string; regressionDraftPath: string; failureClustersPath: string; } { return { suitePath: path.join(outDir, "suite.json"), merchantPath: path.join(outDir, "merchant.json"), intakeSummaryPath: path.join(outDir, "intake-summary.md"), reportJsonPath: path.join(outDir, "report.json"), reportHtmlPath: path.join(outDir, "report.html"), runSummaryPath: path.join(outDir, "summary.md"), junitPath: path.join(outDir, "junit.xml"), commercialPath: path.join(outDir, "commercial-report.md"), recapPath: path.join(outDir, "pilot-recap.md"), proofCardPath: path.join(outDir, "proof-card.md"), regressionDraftPath: path.join(outDir, "regression-draft.json"), failureClustersPath: path.join(outDir, "failure-clusters.md"), }; } function parseTranscriptIntakeArgs(argv: string[]): TranscriptIntakeArgs { const values = new Map(); const flags = new Set(); const knownValues = new Set([ "transcript", "input", "suite", "out", "merchant", "merchant-out", "summary", "merchant-name", "industry", "name", "scenario-id", "scenario-title", "source", "intake", "turn-role", ]); for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (!arg.startsWith("--")) { throw new Error(`Unexpected argument: ${arg}`); } const name = arg.slice(2); if (name === "stdin") { flags.add(name); continue; } if (!knownValues.has(name)) { throw new Error(`Unknown transcript-intake option: --${name}`); } const value = argv[index + 1]; if (!value || value.startsWith("--")) { throw new Error(`${arg} requires a value`); } values.set(name, value); index += 1; } const readFromStdin = flags.has("stdin"); const transcriptPath = values.get("transcript") ?? values.get("input"); if (readFromStdin && transcriptPath) { throw new Error("--stdin cannot be combined with --transcript or --input"); } if (!readFromStdin && !transcriptPath) { throw new Error("--transcript, --input, or --stdin is required"); } const suitePath = values.get("suite") ?? values.get("out") ?? ".voice-testops/transcript-intake/suite.json"; const merchantPath = values.get("merchant"); const merchantOutPath = values.get("merchant-out") ?? (merchantPath ? undefined : ".voice-testops/transcript-intake/merchant.json"); const summaryPath = values.get("summary") ?? ".voice-testops/transcript-intake/summary.md"; const intake = values.get("intake"); return { transcriptPath, readFromStdin, suitePath, merchantPath, merchantOutPath, summaryPath, merchantName: values.get("merchant-name"), industry: values.has("industry") ? industrySchema.parse(values.get("industry")) : undefined, name: values.get("name"), scenarioId: values.get("scenario-id"), scenarioTitle: values.get("scenario-title"), source: leadSourceSchema.parse(values.get("source") ?? "website"), intake: intake ? parseTranscriptIntakePreset(intake) : undefined, turnRole: parseTranscriptTurnRole(values.get("turn-role") ?? "customer"), }; } function parseTranscriptTrialArgs(argv: string[]): TranscriptTrialArgs { const values = new Map(); const flags = new Set(); const knownValues = new Set([ "transcript", "input", "out-dir", "merchant", "merchant-name", "industry", "name", "scenario-id", "scenario-title", "source", "intake", "customer", "period", "proof-url", "fail-on-severity", ]); for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (!arg.startsWith("--")) { throw new Error(`Unexpected argument: ${arg}`); } const name = arg.slice(2); if (name === "stdin") { flags.add(name); continue; } if (!knownValues.has(name)) { throw new Error(`Unknown transcript-trial option: --${name}`); } const value = argv[index + 1]; if (!value || value.startsWith("--")) { throw new Error(`${arg} requires a value`); } values.set(name, value); index += 1; } const readFromStdin = flags.has("stdin"); const transcriptPath = values.get("transcript") ?? values.get("input"); if (readFromStdin && transcriptPath) { throw new Error("--stdin cannot be combined with --transcript or --input"); } if (!readFromStdin && !transcriptPath) { throw new Error("--transcript, --input, or --stdin is required"); } const intake = values.get("intake"); const failOnSeverity = values.get("fail-on-severity"); if ( failOnSeverity !== undefined && failOnSeverity !== "critical" && failOnSeverity !== "major" && failOnSeverity !== "minor" ) { throw new Error("--fail-on-severity must be critical, major, or minor"); } return { transcriptPath, readFromStdin, outDir: values.get("out-dir") ?? ".voice-testops/transcript-trial", merchantPath: values.get("merchant"), merchantName: values.get("merchant-name"), industry: values.has("industry") ? industrySchema.parse(values.get("industry")) : undefined, name: values.get("name"), scenarioId: values.get("scenario-id"), scenarioTitle: values.get("scenario-title"), source: leadSourceSchema.parse(values.get("source") ?? "website"), intake: intake ? parseTranscriptIntakePreset(intake) : undefined, customerName: values.get("customer"), period: values.get("period"), proofUrl: values.get("proof-url"), failOnSeverity, }; } function parsePilotReportArgs(argv: string[]): PilotReportArgs { const values = parseKeyValueArgs(argv); for (const option of values.keys()) { if ( option !== "report" && option !== "commercial" && option !== "recap" && option !== "customer" && option !== "period" ) { throw new Error(`Unknown pilot-report option: --${option}`); } } const reportPath = values.get("report"); if (!reportPath) { throw new Error("--report is required"); } const commercialPath = values.get("commercial"); const recapPath = values.get("recap"); if (!commercialPath && !recapPath) { throw new Error("--commercial or --recap is required"); } return { reportPath, commercialPath, recapPath, customerName: values.get("customer"), period: values.get("period"), }; } function parseProofCardArgs(argv: string[]): ProofCardArgs { const values = parseKeyValueArgs(argv); for (const option of values.keys()) { if ( option !== "report" && option !== "out" && option !== "customer" && option !== "period" && option !== "proof-url" && option !== "next-ask" ) { throw new Error(`Unknown proof-card option: --${option}`); } } const reportPath = values.get("report"); if (!reportPath) { throw new Error("--report is required"); } return { reportPath, outPath: values.get("out"), customerName: values.get("customer"), period: values.get("period"), proofUrl: values.get("proof-url"), nextAsk: values.get("next-ask"), }; } async function calibrateJudge(argv: string[]): Promise { const args = parseCalibrateJudgeArgs(argv); const annotationSet = parseSemanticJudgeAnnotationSet( JSON.parse(await readFile(await resolveReadablePath(args.seedPath), "utf8")), ); const report = await calibrateSemanticJudge(annotationSet); const markdown = renderSemanticJudgeCalibrationMarkdown(report, { maxExamples: args.maxExamples }); if (args.outPath) { await writeReport(args.outPath, markdown); console.log(`Semantic judge calibration: ${args.outPath}`); } else { process.stdout.write(markdown); } if (args.jsonPath) { await writeReport(args.jsonPath, `${JSON.stringify(report, null, 2)}\n`); console.log(`Semantic judge calibration JSON: ${args.jsonPath}`); } console.log( `Calibration summary: ${report.summary.agreements}/${report.summary.total} agreements (${( report.summary.agreementRate * 100 ).toFixed(1)}%), ${report.summary.disagreements} disagreements`, ); if (args.failOnDisagreement) { return report.summary.disagreements === 0 ? 0 : 1; } return 0; } function parseCalibrateJudgeArgs(argv: string[]): CalibrateJudgeArgs { const values = new Map(); const flags = new Set(); const knownValues = new Set(["seed", "out", "json", "max-examples"]); for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (!arg.startsWith("--")) { throw new Error(`Unexpected argument: ${arg}`); } const name = arg.slice(2); if (name === "fail-on-disagreement") { flags.add(name); continue; } if (!knownValues.has(name)) { throw new Error(`Unknown calibrate-judge option: --${name}`); } const value = argv[index + 1]; if (!value || value.startsWith("--")) { throw new Error(`${arg} requires a value`); } values.set(name, value); index += 1; } const maxExamplesValue = values.get("max-examples"); const maxExamples = maxExamplesValue === undefined ? undefined : Number.parseInt(maxExamplesValue, 10); if (maxExamples !== undefined && (!Number.isInteger(maxExamples) || maxExamples < 0)) { throw new Error("--max-examples must be a non-negative integer"); } return { seedPath: values.get("seed") ?? "examples/voice-testops/annotations/semantic-judge-seed.zh-CN.json", outPath: values.get("out"), jsonPath: values.get("json"), maxExamples, failOnDisagreement: flags.has("fail-on-disagreement"), }; } function safeFileName(value: string): string { const normalized = value.replace(/[^a-zA-Z0-9._-]+/g, "_").replace(/^_+|_+$/g, ""); return normalized || "call"; } function parseDraftRegressionsArgs(argv: string[]): DraftRegressionsArgs { const values = parseKeyValueArgs(argv); for (const option of values.keys()) { if (option !== "report" && option !== "suite" && option !== "out" && option !== "clusters") { throw new Error(`Unknown draft-regressions option: --${option}`); } } const reportPath = values.get("report"); if (!reportPath) { throw new Error("--report is required"); } const suitePath = values.get("suite"); if (!suitePath) { throw new Error("--suite is required"); } const outPath = values.get("out"); if (!outPath) { throw new Error("--out is required"); } return { reportPath, suitePath, outPath, clustersPath: values.get("clusters"), }; } function parseCompareArgs(argv: string[]): CompareArgs { const values = new Map(); const flags = new Set(); for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (!arg.startsWith("--")) { throw new Error(`Unexpected argument: ${arg}`); } const name = arg.slice(2); if (name === "fail-on-new") { flags.add(name); continue; } if (name !== "baseline" && name !== "current" && name !== "diff-markdown" && name !== "fail-on-severity") { throw new Error(`Unknown compare option: --${name}`); } const value = argv[index + 1]; if (!value || value.startsWith("--")) { throw new Error(`${arg} requires a value`); } values.set(name, value); index += 1; } const baselinePath = values.get("baseline"); if (!baselinePath) { throw new Error("--baseline is required"); } const currentPath = values.get("current"); if (!currentPath) { throw new Error("--current is required"); } const failOnSeverity = values.get("fail-on-severity"); if ( failOnSeverity !== undefined && failOnSeverity !== "critical" && failOnSeverity !== "major" && failOnSeverity !== "minor" ) { throw new Error("--fail-on-severity must be critical, major, or minor"); } return { baselinePath, currentPath, diffMarkdownPath: values.get("diff-markdown"), failOnNew: flags.has("fail-on-new"), failOnSeverity, }; } async function createAgentFromArgs(args: ReturnType) { if (args.agent === "http") { return createHttpAgent({ endpoint: args.endpoint ?? "" }); } if (args.agent === "openclaw") { return createOpenClawAgent({ endpoint: args.endpoint ?? "", apiKey: args.apiKey, mode: args.openClawMode }); } if (args.agent === "sip") { return createSipAgent({ driverCommand: args.sipDriverCommand ?? "", sipUri: args.sipUri ?? "", sipProxy: args.sipProxy, sipFrom: args.sipFrom, callTimeoutMs: args.sipCallTimeoutMs, driverRetries: args.sipDriverRetries, mediaDir: args.sipMediaDir, }); } if (args.agent === "transcript") { const transcript = await readFile(await resolveReadablePath(args.transcriptPath ?? ""), "utf8"); return createTranscriptReplayAgent({ transcript }); } return createLocalReceptionistAgent(); } async function writeReport(filePath: string, content: string): Promise { await mkdir(path.dirname(filePath), { recursive: true }); await writeFile(filePath, content, "utf8"); } async function readVoiceTestReport(filePath: string, label: string): Promise { const content = await readFile(filePath, "utf8"); const parsed = JSON.parse(content) as Partial; if (!parsed.suiteName || !parsed.summary || !Array.isArray(parsed.scenarios)) { throw new Error(`${label} report is not a Voice Agent TestOps JSON report: ${filePath}`); } return parsed as VoiceTestRunResult; } async function generateSuiteFromTranscript(argv: string[]): Promise { const args = parseFromTranscriptArgs(argv); const transcript = args.readFromStdin ? await readFromStdin() : await readFile(await resolveReadablePath(args.transcriptPath ?? ""), "utf8"); const intakeDefaults = args.intake ? getTranscriptIntakeDefaults(args.intake) : undefined; const merchantName = args.merchantName ?? intakeDefaults?.merchantName; const industry = args.industry ?? intakeDefaults?.industry; const merchant = args.merchantPath ? merchantConfigSchema.parse(JSON.parse(await readFile(await resolveReadablePath(args.merchantPath), "utf8"))) : buildDraftMerchantFromTranscript({ transcript, name: merchantName, industry, }); const suite = buildVoiceTestSuiteFromTranscript({ transcript, merchant, name: args.name ?? intakeDefaults?.suiteName, scenarioId: args.scenarioId ?? intakeDefaults?.scenarioId, scenarioTitle: args.scenarioTitle ?? intakeDefaults?.scenarioTitle, source: args.source, turnRole: args.turnRole, }); const suiteOutput = args.merchantOutPath ? buildSuiteWithMerchantRef( suite, args.outPath ? relativeMerchantRef(args.outPath, args.merchantOutPath) : args.merchantOutPath, ) : suite; const appendCounts = args.append && args.outPath ? await countAppendScenarios(args.outPath, suiteOutput) : undefined; if (args.preview) { printFromTranscriptPreview({ args, suite, merchant, appendCounts, }); return 0; } const output = args.append ? await appendGeneratedScenarios(args.outPath ?? "", suiteOutput) : suiteOutput; if (args.printJson) { process.stdout.write(`${JSON.stringify(output, null, 2)}\n`); return 0; } if (args.merchantOutPath) { await writeReport(args.merchantOutPath, `${JSON.stringify(merchant, null, 2)}\n`); } await writeReport(args.outPath ?? "", `${JSON.stringify(output, null, 2)}\n`); console.log(`${args.append ? "Appended scenario" : "Generated suite"}: ${args.outPath}`); if (args.readFromStdin) { console.log("Transcript: read from stdin"); } if (args.merchantOutPath) { console.log(`${args.merchantPath ? "Merchant profile" : "Merchant draft"}: ${args.merchantOutPath}`); } if (args.intake) { console.log(`Transcript intake: ${args.intake}`); } console.log(`Suite: ${suite.name}`); console.log(`Scenario: ${suite.scenarios[0].id} - ${suite.scenarios[0].title}`); if (!args.merchantPath && !args.merchantOutPath) { console.log("Merchant draft: generated from transcript"); } printTurnCount(args.turnRole, suite.scenarios[0].turns.length); return 0; } type FromTranscriptTurnRole = "customer" | "assistant"; type FromTranscriptArgs = { transcriptPath?: string; readFromStdin: boolean; merchantPath?: string; merchantOutPath?: string; append: boolean; preview: boolean; printJson: boolean; outPath?: string; merchantName?: string; industry?: Industry; name?: string; scenarioId?: string; scenarioTitle?: string; source: LeadSource; intake?: TranscriptIntakePreset; turnRole: FromTranscriptTurnRole; }; function parseFromTranscriptArgs(argv: string[]): FromTranscriptArgs { const values = new Map(); const flags = new Set(); const knownValues = new Set([ "transcript", "input", "merchant", "merchant-out", "out", "merchant-name", "industry", "name", "scenario-id", "scenario-title", "source", "intake", "turn-role", ]); for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (!arg.startsWith("--")) { throw new Error(`Unexpected argument: ${arg}`); } const name = arg.slice(2); if (name === "stdin" || name === "append" || name === "preview" || name === "print-json") { flags.add(name); continue; } if (!knownValues.has(name)) { throw new Error(`Unknown from-transcript option: --${name}`); } const value = argv[index + 1]; if (!value || value.startsWith("--")) { throw new Error(`${arg} requires a value`); } values.set(name, value); index += 1; } const readFromStdin = flags.has("stdin"); const transcriptPath = values.get("transcript") ?? values.get("input"); const merchantPath = values.get("merchant"); const merchantOutPath = values.get("merchant-out"); const append = flags.has("append"); const preview = flags.has("preview"); const printJson = flags.has("print-json"); const outPath = values.get("out"); if (readFromStdin && transcriptPath) { throw new Error("--stdin cannot be combined with --transcript or --input"); } if (!readFromStdin && !transcriptPath) { throw new Error("--transcript, --input, or --stdin is required"); } if (preview && printJson) { throw new Error("--preview cannot be combined with --print-json"); } if (!preview && !printJson && !outPath) { throw new Error("--out is required"); } if (append && !outPath) { throw new Error("--append requires --out"); } const source = leadSourceSchema.parse(values.get("source") ?? "website"); const intake = values.get("intake"); const turnRole = parseTranscriptTurnRole(values.get("turn-role") ?? "customer"); return { transcriptPath, readFromStdin, merchantPath, merchantOutPath, append, preview, printJson, outPath, merchantName: values.get("merchant-name"), industry: values.has("industry") ? industrySchema.parse(values.get("industry")) : undefined, name: values.get("name"), scenarioId: values.get("scenario-id"), scenarioTitle: values.get("scenario-title"), source, intake: intake ? parseTranscriptIntakePreset(intake) : undefined, turnRole, }; } function parseTranscriptTurnRole(value: string): FromTranscriptTurnRole { if (value === "customer" || value === "assistant") { return value; } throw new Error("--turn-role must be customer or assistant"); } type AppendCounts = { existing: number; added: number; }; async function countAppendScenarios(suitePath: string, generatedSuite: unknown): Promise { const existingSuite = await readRawSuiteWithScenarios(suitePath); if (!isJsonObject(generatedSuite) || !Array.isArray(generatedSuite.scenarios)) { throw new Error("Generated suite must include a scenarios array"); } return { existing: existingSuite.scenarios.length, added: generatedSuite.scenarios.length, }; } async function appendGeneratedScenarios(suitePath: string, generatedSuite: unknown): Promise { const existingSuite = await readRawSuiteWithScenarios(suitePath); if (!isJsonObject(generatedSuite) || !Array.isArray(generatedSuite.scenarios)) { throw new Error("Generated suite must include a scenarios array"); } return { ...existingSuite, scenarios: [...existingSuite.scenarios, ...generatedSuite.scenarios], }; } async function readRawSuiteWithScenarios(suitePath: string): Promise & { scenarios: unknown[] }> { const existingSuite = JSON.parse(await readFile(await resolveReadablePath(suitePath), "utf8")) as unknown; if (!isJsonObject(existingSuite) || !Array.isArray(existingSuite.scenarios)) { throw new Error(`Existing suite must include a scenarios array: ${suitePath}`); } return existingSuite as Record & { scenarios: unknown[] }; } function buildSuiteWithMerchantRef(suite: VoiceTestSuite, merchantRef: string): unknown { return { ...suite, scenarios: suite.scenarios.map(({ merchant: _merchant, ...scenario }) => ({ ...scenario, merchantRef, })), }; } function isJsonObject(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function relativeMerchantRef(suitePath: string, merchantPath: string): string { return path.relative(path.dirname(suitePath), merchantPath).split(path.sep).join(path.posix.sep); } function printFromTranscriptPreview(options: { args: FromTranscriptArgs; suite: VoiceTestSuite; merchant: { name: string; industry: string }; appendCounts?: AppendCounts; }): void { const scenario = options.suite.scenarios[0]; const assertions = scenario.turns.reduce((count, turn) => count + turn.expect.length, 0); console.log("Preview: no files written"); console.log( options.args.append && options.args.outPath ? `Action: append to ${options.args.outPath}` : "Action: generate new suite", ); if (options.args.readFromStdin) { console.log("Transcript: read from stdin"); } if (options.appendCounts) { console.log(`Existing scenarios: ${options.appendCounts.existing}`); console.log(`Result scenarios: ${options.appendCounts.existing + options.appendCounts.added}`); } console.log(`Suite: ${options.suite.name}`); if (options.args.intake) { console.log(`Transcript intake: ${options.args.intake}`); } console.log(`Scenario: ${scenario.id} - ${scenario.title}`); console.log(`Merchant: ${options.merchant.name} (${options.merchant.industry})`); if (options.args.merchantOutPath) { console.log(`Merchant draft target: ${options.args.merchantOutPath}`); } printTurnCount(options.args.turnRole, scenario.turns.length); console.log(`Assertions: ${assertions}`); } function printTurnCount(turnRole: FromTranscriptTurnRole, count: number): void { if (turnRole === "assistant") { console.log("Turn role: assistant"); console.log(`Assistant turns: ${count}`); return; } console.log(`Customer turns: ${count}`); } async function readFromStdin(): Promise { return new Promise((resolve, reject) => { let content = ""; process.stdin.setEncoding("utf8"); process.stdin.on("data", (chunk: string) => { content += chunk; }); process.stdin.on("error", reject); process.stdin.on("end", () => resolve(content)); }); } function parseKeyValueArgs(argv: string[]): Map { const values = new Map(); for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (!arg.startsWith("--")) { throw new Error(`Unexpected argument: ${arg}`); } const value = argv[index + 1]; if (!value || value.startsWith("--")) { throw new Error(`${arg} requires a value`); } values.set(arg.slice(2), value); index += 1; } return values; } function countFailuresAtOrAboveSeverity(result: VoiceTestRunResult, threshold: VoiceTestSeverity): number { const thresholdRank = severityRank[threshold]; return result.scenarios.reduce( (scenarioCount, scenario) => scenarioCount + scenario.turns.reduce( (turnCount, turn) => turnCount + turn.failures.filter((failure) => severityRank[failure.severity] >= thresholdRank).length, 0, ), 0, ); } function countFailureRecordsAtOrAboveSeverity( failures: Array<{ severity: VoiceTestSeverity }>, threshold: VoiceTestSeverity, ): number { const thresholdRank = severityRank[threshold]; return failures.filter((failure) => severityRank[failure.severity] >= thresholdRank).length; } function formatNewFailureGate(newFailures: number, threshold: VoiceTestSeverity | undefined): string { if (!threshold) { return `New failure gate: ${newFailures === 0 ? "passed" : "failed"} (${newFailures} new failures)`; } return `New failure gate: ${newFailures === 0 ? "passed" : "failed"} (${newFailures} new failures at or above ${threshold})`; } main(process.argv.slice(2)) .then((code) => { process.exitCode = code; }) .catch((error: unknown) => { console.error(error instanceof Error ? error.message : error); process.exitCode = 1; });