import { expectBufferedAudioAtLeastOnce, expectEvent, expectNoEvent, expectStatusTransition, } from "../fixtures/assertions"; import { compareAuditResults, deriveAuditMetrics, type ScenarioAuditMetrics, } from "../fixtures/metrics"; import { expect, test } from "../fixtures/resilienceHarness"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; const productionLikeResilienceConfig = { connectTimeoutMs: 5000, healthCheckIntervalMs: 500, reconnectBaseDelayMs: 250, reconnectMaxDelayMs: 2000, reconnectDelayGrowFactor: 1.5, minConnectionUptimeMs: 1000, maxReconnectAttempts: 8, }; const auditStressResilienceConfig = { connectTimeoutMs: 250, healthCheckIntervalMs: 50, reconnectBaseDelayMs: 50, reconnectMaxDelayMs: 100, reconnectDelayGrowFactor: 1.2, minConnectionUptimeMs: 0, maxReconnectAttempts: 4, }; type AuditNetworkAction = | { atMs: number; kind: "offline"; durationMs: number; } | { atMs: number; kind: "terminateSockets"; }; type AuditServerMessage = { atMs: number; kind: "final" | "partial"; text: string; }; type AuditScenario = { id: string; speechPlan: Array<{ kind: "speech" | "silence"; durationMs: number }>; networkPlan: AuditNetworkAction[]; serverMessages: AuditServerMessage[]; }; const defaultSpeechPlan = [ { kind: "speech" as const, durationMs: 900 }, { kind: "silence" as const, durationMs: 150 }, { kind: "speech" as const, durationMs: 900 }, { kind: "silence" as const, durationMs: 150 }, { kind: "speech" as const, durationMs: 900 }, ]; const mergeGatingScenarios: AuditScenario[] = [ { id: "baseline_clean_session", speechPlan: defaultSpeechPlan, networkPlan: [], serverMessages: [ { atMs: 350, kind: "final", text: "baseline phrase one" }, { atMs: 1250, kind: "final", text: "baseline phrase two" }, { atMs: 2350, kind: "final", text: "baseline phrase three" }, ], }, { id: "short_offline_flap", speechPlan: [ { kind: "speech", durationMs: 1200 }, { kind: "silence", durationMs: 150 }, { kind: "speech", durationMs: 1200 }, { kind: "silence", durationMs: 150 }, { kind: "speech", durationMs: 1200 }, ], networkPlan: [{ atMs: 900, kind: "offline", durationMs: 700 }], serverMessages: [ { atMs: 250, kind: "final", text: "phrase before outage" }, { atMs: 2000, kind: "final", text: "phrase recovered after outage" }, { atMs: 2700, kind: "final", text: "phrase after recovery" }, ], }, { id: "repeated_short_flaps", speechPlan: [ { kind: "speech", durationMs: 1000 }, { kind: "silence", durationMs: 120 }, { kind: "speech", durationMs: 1000 }, { kind: "silence", durationMs: 120 }, { kind: "speech", durationMs: 1000 }, { kind: "silence", durationMs: 120 }, { kind: "speech", durationMs: 1000 }, ], networkPlan: [ { atMs: 700, kind: "offline", durationMs: 350 }, { atMs: 1700, kind: "offline", durationMs: 350 }, { atMs: 2700, kind: "offline", durationMs: 350 }, ], serverMessages: [ { atMs: 300, kind: "final", text: "repeated flap phrase one" }, { atMs: 1400, kind: "final", text: "repeated flap phrase two" }, { atMs: 2400, kind: "final", text: "repeated flap phrase three" }, { atMs: 3500, kind: "final", text: "repeated flap phrase four" }, ], }, { id: "long_outage_recover_after_speech", speechPlan: [ { kind: "speech", durationMs: 1500 }, { kind: "silence", durationMs: 100 }, { kind: "speech", durationMs: 1500 }, { kind: "silence", durationMs: 100 }, { kind: "speech", durationMs: 1500 }, ], networkPlan: [{ atMs: 700, kind: "offline", durationMs: 2400 }], serverMessages: [ { atMs: 250, kind: "final", text: "before long outage" }, { atMs: 3400, kind: "final", text: "buffered segment recovered" }, { atMs: 4100, kind: "final", text: "final segment after long outage" }, ], }, { id: "abrupt_transport_resets", speechPlan: [ { kind: "speech", durationMs: 1100 }, { kind: "silence", durationMs: 100 }, { kind: "speech", durationMs: 1100 }, { kind: "silence", durationMs: 100 }, { kind: "speech", durationMs: 1100 }, ], networkPlan: [ { atMs: 600, kind: "terminateSockets" }, { atMs: 1800, kind: "terminateSockets" }, ], serverMessages: [ { atMs: 300, kind: "final", text: "before reset" }, { atMs: 2200, kind: "final", text: "after first reset" }, { atMs: 3100, kind: "final", text: "after second reset" }, ], }, ]; const waitUntilOffset = async (startedAt: number, targetOffsetMs: number) => { const remainingMs = targetOffsetMs - (Date.now() - startedAt); if (remainingMs > 0) { await new Promise((resolve) => setTimeout(resolve, remainingMs)); } }; const waitForFinishSafely = async (scriptedServer: { waitForFinish: (timeoutMs?: number) => Promise; }) => { try { await scriptedServer.waitForFinish(3_000); return true; } catch { return false; } }; const auditBaselinePath = path.join( process.cwd(), "e2e/fixtures/resilience-audit.baseline.json" ); const readJsonFile = async (filePath: string): Promise => { try { return JSON.parse(await readFile(filePath, "utf8")) as T; } catch { return null; } }; const runAuditScenario = async ({ context, harness, scriptedServer, scenario, resilience, }: { context: Parameters[0]["context"]; harness: Parameters[0]["harness"]; scriptedServer: Parameters[0]["scriptedServer"]; scenario: AuditScenario; resilience: typeof productionLikeResilienceConfig | typeof auditStressResilienceConfig; }) => { await context.setOffline(false); scriptedServer.clearTransportLog(); await harness.start({ resilience, speechPlan: scenario.speechPlan, }); await scriptedServer.waitForConnectionCount(1); await scriptedServer.waitForBinaryFrames(1); const startedAt = Date.now(); const executedFaults: number[] = []; for (const message of scenario.serverMessages) { void (async () => { await waitUntilOffset(startedAt, message.atMs); if (message.kind === "partial") { scriptedServer.sendPartial(message.text); return; } scriptedServer.sendFinal(message.text); })(); } for (const action of scenario.networkPlan) { await waitUntilOffset(startedAt, action.atMs); executedFaults.push(Date.now()); if (action.kind === "offline") { await context.setOffline(true); scriptedServer.terminateActiveSockets(); await new Promise((resolve) => setTimeout(resolve, action.durationMs)); await context.setOffline(false); continue; } scriptedServer.terminateActiveSockets(); } const scenarioDurationMs = Math.max( scenario.speechPlan.reduce((total, segment) => total + segment.durationMs, 0), ...scenario.serverMessages.map((message) => message.atMs + 150) ); await waitUntilOffset(startedAt, scenarioDurationMs + 1200); await harness.stop(); await waitForFinishSafely(scriptedServer); const events = await harness.getEvents(); const statuses = await harness.getStatuses(); const finals = await harness.getFinals(); const transportLog = scriptedServer.getTransportLog(); await context.setOffline(false); return { events, statuses, finals, transportLog, metrics: deriveAuditMetrics({ scenarioId: scenario.id, events, statuses, finals, transportLog, firstFaultAt: executedFaults[0] ?? null, }), }; }; test.describe("Resilience browser scaffold", () => { test.describe("Merge-gating resilience cases", () => { test("R001 baseline happy path uses the harness without recovery", async ({ harness, scriptedServer, }) => { await harness.start({ resilience: productionLikeResilienceConfig, }); await scriptedServer.waitForConnectionCount(1); await scriptedServer.waitForBinaryFrames(1); scriptedServer.sendPartial("phrase one partial"); scriptedServer.sendFinal("phrase one final"); scriptedServer.sendFinal("phrase two final"); scriptedServer.sendFinal("phrase three final"); await harness.stop(); await scriptedServer.waitForFinish(); const events = await harness.getEvents(); const statuses = await harness.getStatuses(); const finals = await harness.getFinals(); const transportLog = scriptedServer.getTransportLog(); expectEvent(events, "connected"); expectNoEvent(events, "reconnecting"); expectNoEvent(events, "disconnected"); expect(finals).toEqual([ "phrase one final", "phrase two final", "phrase three final", ]); expectStatusTransition(statuses, "connected"); expectStatusTransition(statuses, "stopped"); expect( transportLog.some((entry) => entry.type === "finish") ).toBeTruthy(); }); test("R002 short offline flap emits recovery events and buffers audio", async ({ context, harness, scriptedServer, }) => { await harness.start({ resilience: productionLikeResilienceConfig, speechPlan: [ { kind: "speech", durationMs: 1200 }, { kind: "silence", durationMs: 150 }, { kind: "speech", durationMs: 1200 }, { kind: "silence", durationMs: 150 }, { kind: "speech", durationMs: 1200 }, ], }); await scriptedServer.waitForConnectionCount(1); scriptedServer.sendFinal("phrase before outage"); await context.setOffline(true); scriptedServer.terminateActiveSockets(); await harness.waitForEvent("reconnecting"); await new Promise((resolve) => setTimeout(resolve, 400)); await context.setOffline(false); await scriptedServer.waitForConnectionCount(2); await harness.waitForEvent("reconnected"); scriptedServer.sendFinal("phrase during outage recovered"); scriptedServer.sendFinal("phrase after recovery"); await harness.stop(); await scriptedServer.waitForFinish(); const events = await harness.getEvents(); const statuses = await harness.getStatuses(); const transcript = await harness.getTranscript(); expectEvent(events, "connected"); expectEvent(events, "reconnecting"); expectEvent(events, "reconnected"); expectNoEvent(events, "disconnected"); expectStatusTransition(statuses, "reconnecting"); expectStatusTransition(statuses, "connected"); expectStatusTransition(statuses, "stopped"); expectBufferedAudioAtLeastOnce(statuses); expect(transcript).toContain("phrase before outage"); expect(transcript).toContain("phrase during outage recovered"); expect(transcript).toContain("phrase after recovery"); }); test.skip("R003 multiple short flaps in the same session", async () => { // Scaffold placeholder: // repeat R002 three times and assert multiple recoveries in one session. }); test.skip( "R004 long outage while user keeps speaking, recover only after speech ends, then finalize", async () => { // Scaffold placeholder: // keep the browser offline for most of the speech plan and assert backlog drain after recovery. } ); }); test.skip("R005 retry budget exhaustion becomes terminal disconnect", async () => { // Scaffold placeholder: // reduce maxReconnectAttempts, keep the transport unavailable, and assert terminal disconnected state. }); test.skip("R006 recovery after websocket upstream backpressure stall", async () => { // Scaffold placeholder: // simulate stalled upstream drain and assert reconnect due to stalled bufferedAmount growth. }); test.skip("R007 stop during recovering state", async () => { // Scaffold placeholder: // click stop while reconnecting and assert clean shutdown with no further retries. }); test.skip("R008 pause and resume after recovery", async () => { // Scaffold placeholder: // pause after a recovered outage, resume, and assert websocket refresh with buffered audio before reconnect. }); test.skip("R009 long outage large enough to require persisted offline segments", async () => { // Scaffold placeholder: // hold outage long enough to exceed OFFLINE_AUDIO_SEGMENT_BYTES and assert persisted buffer growth and drain. }); test.skip("R010 new session clears previous buffered leftovers", async () => { // Scaffold placeholder: // force session A buffering, then start session B and assert a clean buffer state. }); test.skip("R011 stop after long outage without prior online recovery", async () => { // Scaffold placeholder: // stop while offline and assert timeout-based completion fallback. }); test.skip("R012 flaky reconnect with short-lived successful opens", async () => { // Scaffold placeholder: // simulate unstable opens below minConnectionUptimeMs and assert retry progression. }); test.skip("R013 high latency and low throughput without full offline mode", async () => { // Scaffold placeholder: // use Chromium CDP network emulation for degraded-but-not-offline transport behavior. }); test.skip("R014 backend closes socket unexpectedly after receiving buffered backlog", async () => { // Scaffold placeholder: // fail again during backlog drain and assert buffered audio survives a second recovery cycle. }); test.skip("R015 recovery status contract", async () => { // Scaffold placeholder: // validate the resilience_status snapshot as a public API contract. }); test.describe("Audit sweep", () => { test("R100 resilience audit matrix collects first critical points", async ({ context, harness, scriptedServer, }, testInfo) => { test.setTimeout(90_000); const results: ScenarioAuditMetrics[] = []; for (const scenario of mergeGatingScenarios) { const { metrics } = await runAuditScenario({ context, harness, scriptedServer, scenario, resilience: auditStressResilienceConfig, }); results.push(metrics); } const baseline = await readJsonFile<{ label?: string; generatedAt?: string; results: ScenarioAuditMetrics[]; }>(auditBaselinePath); const baselineComparison = baseline ? compareAuditResults(results, baseline.results) : null; if (baselineComparison) { baselineComparison.summary.baselineLabel = baseline.label ?? baseline.generatedAt ?? "resilience-audit.baseline.json"; } const auditDir = path.join(process.cwd(), "test-results"); await mkdir(auditDir, { recursive: true }); const auditPath = path.join(auditDir, "resilience-audit.json"); await writeFile( auditPath, JSON.stringify( { generatedAt: new Date().toISOString(), project: testInfo.project.name, results, baselineComparison, }, null, 2 ) ); console.table( results.map((result) => ({ scenario: result.scenarioId, reconnects: result.reconnectEvents, recovered: result.reconnectedEvents, maxBufferedAudioBytes: result.maxBufferedAudioBytes, timeToBufferingMs: result.timeToBufferingMs, timeToRecoveryMs: result.timeToRecoveryMs, timeToDrainAfterRecoveryMs: result.timeToDrainAfterRecoveryMs, flags: result.flags.join(","), })) ); if (baselineComparison) { console.table({ baseline: baselineComparison.summary.baselineLabel, terminalDisconnectCount: `${baselineComparison.summary.terminalDisconnectCount.current}/${baselineComparison.summary.terminalDisconnectCount.baseline}`, finishDeliveryRate: `${baselineComparison.summary.finishDeliveryRate.current.toFixed(2)}/${baselineComparison.summary.finishDeliveryRate.baseline.toFixed(2)}`, drainCompletionRate: `${baselineComparison.summary.drainCompletionRate.current.toFixed(2)}/${baselineComparison.summary.drainCompletionRate.baseline.toFixed(2)}`, medianRecoveryTimeMs: `${baselineComparison.summary.medianRecoveryTimeMs.current ?? "null"}/${baselineComparison.summary.medianRecoveryTimeMs.baseline ?? "null"}`, maxBufferedAudioBytes: `${baselineComparison.summary.maxBufferedAudioBytes.current}/${baselineComparison.summary.maxBufferedAudioBytes.baseline}`, regressions: baselineComparison.regressions.length, }); } expect(results).toHaveLength(mergeGatingScenarios.length); expect( results.every((result) => result.binaryFramesReceived >= 1) ).toBeTruthy(); if ( baselineComparison && process.env.RESILIENCE_AUDIT_ENFORCE_BASELINE === "1" ) { expect(baselineComparison.regressions).toHaveLength(0); } }); }); });