type HarnessEvent = { name: string; detail: unknown; timestamp: number; }; type ResilienceStatus = { connectionState?: string; websocketState?: string; totalBufferedAudioBytes?: number; [key: string]: unknown; timestamp: number; }; type TransportLogEntry = | { type: "connection-open"; timestamp: number; connectionId: number; } | { type: "connection-close"; timestamp: number; connectionId: number; code: number; reason: string; } | { type: "binary-frame"; timestamp: number; connectionId: number; byteLength: number; } | { type: "text-frame"; timestamp: number; connectionId: number; text: string; } | { type: "finish"; timestamp: number; connectionId: number; } | { type: "server-message"; timestamp: number; payload: Record; }; export type ScenarioAuditMetrics = { scenarioId: string; sessionMs: number; reconnectEvents: number; reconnectedEvents: number; disconnectedEvents: number; connectionOpenCount: number; connectionCloseCount: number; binaryFramesReceived: number; finishMessagesReceived: number; finalsCount: number; duplicateFinals: number; maxBufferedAudioBytes: number; bufferedAudioObserved: boolean; drainedAfterRecovery: boolean; timeToBufferingMs: number | null; timeToRecoveryMs: number | null; timeToDrainAfterRecoveryMs: number | null; flags: string[]; }; export type AuditBaselineFile = { label?: string; generatedAt?: string; results: ScenarioAuditMetrics[]; }; export type AuditRegression = { scenarioId: string; metric: string; baseline: number | boolean | string | null; current: number | boolean | string | null; reason: string; }; export type AuditComparisonSummary = { baselineLabel: string | null; comparedScenarios: number; terminalDisconnectCount: { baseline: number; current: number; }; finishDeliveryRate: { baseline: number; current: number; }; drainCompletionRate: { baseline: number; current: number; }; medianRecoveryTimeMs: { baseline: number | null; current: number | null; }; maxBufferedAudioBytes: { baseline: number; current: number; }; }; export type AuditComparison = { baselineAvailable: boolean; regressions: AuditRegression[]; summary: AuditComparisonSummary; }; type DeriveAuditMetricsInput = { scenarioId: string; events: HarnessEvent[]; statuses: ResilienceStatus[]; finals: string[]; transportLog: TransportLogEntry[]; firstFaultAt: number | null; }; const countByName = (events: HarnessEvent[], name: string) => events.filter((event) => event.name === name).length; const firstEventAt = (events: HarnessEvent[], name: string) => events.find((event) => event.name === name)?.timestamp ?? null; const maxBufferedAudioBytes = (statuses: ResilienceStatus[]) => statuses.reduce( (max, status) => Math.max(max, status.totalBufferedAudioBytes || 0), 0 ); const median = (values: number[]) => { if (values.length === 0) { return null; } const sorted = values.slice().sort((left, right) => left - right); const middle = Math.floor(sorted.length / 2); if (sorted.length % 2 === 0) { return Math.round((sorted[middle - 1] + sorted[middle]) / 2); } return sorted[middle]; }; const scenarioById = (results: ScenarioAuditMetrics[]) => new Map(results.map((result) => [result.scenarioId, result])); const drainCompletionRate = (results: ScenarioAuditMetrics[]) => { const drained = results.filter((result) => result.bufferedAudioObserved ? result.drainedAfterRecovery : true ).length; return results.length === 0 ? 0 : drained / results.length; }; const finishDeliveryRate = (results: ScenarioAuditMetrics[]) => { const delivered = results.filter((result) => result.finishMessagesReceived > 0).length; return results.length === 0 ? 0 : delivered / results.length; }; const terminalDisconnectCount = (results: ScenarioAuditMetrics[]) => results.filter((result) => result.disconnectedEvents > 0).length; const maxBufferedAudioBytesAcrossResults = (results: ScenarioAuditMetrics[]) => results.reduce((max, result) => Math.max(max, result.maxBufferedAudioBytes), 0); const recoveryTimes = (results: ScenarioAuditMetrics[]) => results .map((result) => result.timeToRecoveryMs) .filter((value): value is number => typeof value === "number"); const allowedNumericRegression = (baseline: number) => Math.max(Math.round(baseline * 0.25), 250); const allowedBufferRegression = (baseline: number) => Math.max(Math.round(baseline * 0.2), 8192); const compareMetricIncrease = ( current: number, baseline: number, metric: string, scenarioId: string, reason: string, regressions: AuditRegression[] ) => { if (current > baseline) { regressions.push({ scenarioId, metric, baseline, current, reason, }); } }; const compareMetricDecrease = ( current: number, baseline: number, metric: string, scenarioId: string, reason: string, regressions: AuditRegression[] ) => { if (current < baseline) { regressions.push({ scenarioId, metric, baseline, current, reason, }); } }; export const compareAuditResults = ( currentResults: ScenarioAuditMetrics[], baselineResults: ScenarioAuditMetrics[] ): AuditComparison => { const baselineMap = scenarioById(baselineResults); const regressions: AuditRegression[] = []; for (const current of currentResults) { const baseline = baselineMap.get(current.scenarioId); if (!baseline) { regressions.push({ scenarioId: current.scenarioId, metric: "baseline", baseline: null, current: null, reason: "Scenario missing from regression baseline.", }); continue; } compareMetricIncrease( current.disconnectedEvents, baseline.disconnectedEvents, "disconnectedEvents", current.scenarioId, "Terminal disconnects increased versus baseline.", regressions ); compareMetricDecrease( current.finishMessagesReceived, baseline.finishMessagesReceived, "finishMessagesReceived", current.scenarioId, "Session completion acknowledgements regressed.", regressions ); compareMetricDecrease( current.reconnectedEvents, baseline.reconnectedEvents, "reconnectedEvents", current.scenarioId, "Recoverable outages are not recovering as often as the baseline.", regressions ); compareMetricDecrease( current.finalsCount, baseline.finalsCount, "finalsCount", current.scenarioId, "Final transcript coverage regressed.", regressions ); compareMetricIncrease( current.duplicateFinals, baseline.duplicateFinals, "duplicateFinals", current.scenarioId, "Duplicate final transcripts increased.", regressions ); compareMetricIncrease( current.maxBufferedAudioBytes, baseline.maxBufferedAudioBytes + allowedBufferRegression(baseline.maxBufferedAudioBytes), "maxBufferedAudioBytes", current.scenarioId, "Buffered audio grew beyond the baseline tolerance.", regressions ); if ( current.timeToRecoveryMs !== null && baseline.timeToRecoveryMs !== null && current.timeToRecoveryMs > baseline.timeToRecoveryMs + allowedNumericRegression(baseline.timeToRecoveryMs) ) { regressions.push({ scenarioId: current.scenarioId, metric: "timeToRecoveryMs", baseline: baseline.timeToRecoveryMs, current: current.timeToRecoveryMs, reason: "Recovery latency regressed beyond tolerance.", }); } if ( baseline.drainedAfterRecovery && !current.drainedAfterRecovery && current.bufferedAudioObserved ) { regressions.push({ scenarioId: current.scenarioId, metric: "drainedAfterRecovery", baseline: baseline.drainedAfterRecovery, current: current.drainedAfterRecovery, reason: "Buffer drain succeeded in baseline but not in current run.", }); } const baselineFlags = new Set(baseline.flags); const newFlags = current.flags.filter((flag) => !baselineFlags.has(flag)); if (newFlags.length > 0) { regressions.push({ scenarioId: current.scenarioId, metric: "flags", baseline: baseline.flags.join(","), current: current.flags.join(","), reason: `New audit flags appeared: ${newFlags.join(", ")}`, }); } } return { baselineAvailable: baselineResults.length > 0, regressions, summary: { baselineLabel: null, comparedScenarios: currentResults.length, terminalDisconnectCount: { baseline: terminalDisconnectCount(baselineResults), current: terminalDisconnectCount(currentResults), }, finishDeliveryRate: { baseline: finishDeliveryRate(baselineResults), current: finishDeliveryRate(currentResults), }, drainCompletionRate: { baseline: drainCompletionRate(baselineResults), current: drainCompletionRate(currentResults), }, medianRecoveryTimeMs: { baseline: median(recoveryTimes(baselineResults)), current: median(recoveryTimes(currentResults)), }, maxBufferedAudioBytes: { baseline: maxBufferedAudioBytesAcrossResults(baselineResults), current: maxBufferedAudioBytesAcrossResults(currentResults), }, }, }; }; export const summarizeAuditResults = (results: ScenarioAuditMetrics[]) => ({ terminalDisconnectCount: terminalDisconnectCount(results), finishDeliveryRate: finishDeliveryRate(results), drainCompletionRate: drainCompletionRate(results), medianRecoveryTimeMs: median(recoveryTimes(results)), maxBufferedAudioBytes: maxBufferedAudioBytesAcrossResults(results), }); export const deriveAuditMetrics = ({ scenarioId, events, statuses, finals, transportLog, firstFaultAt, }: DeriveAuditMetricsInput): ScenarioAuditMetrics => { const sessionStartAt = events[0]?.timestamp ?? statuses[0]?.timestamp ?? transportLog[0]?.timestamp ?? Date.now(); const combinedTimeline = [...events, ...statuses].sort( (left, right) => left.timestamp - right.timestamp ); const sessionEndAt = combinedTimeline[combinedTimeline.length - 1]?.timestamp ?? transportLog[transportLog.length - 1]?.timestamp ?? sessionStartAt; const reconnectAt = firstEventAt(events, "reconnecting"); const reconnectedAt = firstEventAt(events, "reconnected"); const firstBufferedAt = statuses.find((status) => (status.totalBufferedAudioBytes || 0) > 0)?.timestamp ?? null; const maxBuffered = maxBufferedAudioBytes(statuses); const sawBufferedAudio = maxBuffered > 0; const firstZeroAfterRecovery = reconnectedAt === null ? null : statuses.find( (status) => status.timestamp >= reconnectedAt && (status.totalBufferedAudioBytes || 0) === 0 )?.timestamp ?? null; const duplicates = finals.length - new Set(finals).size; const flags: string[] = []; if (sawBufferedAudio && firstZeroAfterRecovery === null) { flags.push("buffer_not_drained_after_recovery"); } if (duplicates > 0) { flags.push("duplicate_final_transcripts"); } if (countByName(events, "disconnected") > 0) { flags.push("terminal_disconnect_observed"); } if (firstFaultAt !== null && reconnectAt === null) { flags.push("fault_without_reconnect_signal"); } if (firstFaultAt !== null && reconnectedAt === null) { flags.push("fault_without_recovery_signal"); } return { scenarioId, sessionMs: Math.max(sessionEndAt - sessionStartAt, 0), reconnectEvents: countByName(events, "reconnecting"), reconnectedEvents: countByName(events, "reconnected"), disconnectedEvents: countByName(events, "disconnected"), connectionOpenCount: transportLog.filter((entry) => entry.type === "connection-open") .length, connectionCloseCount: transportLog.filter((entry) => entry.type === "connection-close") .length, binaryFramesReceived: transportLog.filter((entry) => entry.type === "binary-frame").length, finishMessagesReceived: transportLog.filter((entry) => entry.type === "finish").length, finalsCount: finals.length, duplicateFinals: duplicates, maxBufferedAudioBytes: maxBuffered, bufferedAudioObserved: sawBufferedAudio, drainedAfterRecovery: sawBufferedAudio ? firstZeroAfterRecovery !== null : true, timeToBufferingMs: firstFaultAt !== null && firstBufferedAt !== null ? Math.max(firstBufferedAt - firstFaultAt, 0) : null, timeToRecoveryMs: firstFaultAt !== null && reconnectedAt !== null ? Math.max(reconnectedAt - firstFaultAt, 0) : null, timeToDrainAfterRecoveryMs: reconnectedAt !== null && firstZeroAfterRecovery !== null ? Math.max(firstZeroAfterRecovery - reconnectedAt, 0) : null, flags, }; };