import { describe, expect, mock, test } from "bun:test"; // Only mock sleep so retries complete instantly; keep real retry logic. // NOTE: We must NOT use `await import()` inside mock.module — it deadlocks // bun's module resolver. Instead, inline the real exports and only replace sleep. const sleepSpy = mock((_ms: number) => Promise.resolve()); mock.module("../util/retry.js", () => { const DEFAULT_MAX_RETRIES = 3; const DEFAULT_BASE_DELAY_MS = 1000; function computeRetryDelay( attempt: number, baseDelayMs = DEFAULT_BASE_DELAY_MS, ): number { const cap = baseDelayMs * Math.pow(2, attempt); const half = cap / 2; return half + Math.random() * half; } function parseRetryAfterMs(value: string): number | undefined { const seconds = Number(value); if (!isNaN(seconds)) { return seconds * 1000; } const dateMs = Date.parse(value); if (!isNaN(dateMs)) { return Math.max(0, dateMs - Date.now()); } return undefined; } function getHttpRetryDelay( response: Response, attempt: number, baseDelayMs = DEFAULT_BASE_DELAY_MS, ): number { const retryAfter = response.headers.get("retry-after"); if (retryAfter) { const parsed = parseRetryAfterMs(retryAfter); if (parsed !== undefined) { return parsed; } } const effectiveBase = attempt === 0 ? baseDelayMs * 2 : baseDelayMs; return Math.max(baseDelayMs, computeRetryDelay(attempt, effectiveBase)); } function isRetryableStatus(status: number): boolean { return status === 429 || status >= 500; } const RETRYABLE_NETWORK_MESSAGE_PATTERNS = [ /socket.*closed unexpectedly/i, /socket hang up/i, ]; function isRetryableNetworkError(error: unknown): boolean { if (!(error instanceof Error)) { return false; } const retryableCodes = new Set([ "ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EPIPE", ]); const code = (error as NodeJS.ErrnoException).code; if (code && retryableCodes.has(code)) { return true; } if (error.cause instanceof Error) { const causeCode = (error.cause as NodeJS.ErrnoException).code; if (causeCode && retryableCodes.has(causeCode)) { return true; } } if (RETRYABLE_NETWORK_MESSAGE_PATTERNS.some((p) => p.test(error.message))) { return true; } const cause = error.cause; if ( cause instanceof Error && RETRYABLE_NETWORK_MESSAGE_PATTERNS.some((p) => p.test(cause.message)) ) { return true; } return false; } function extractRetryAfterMs(headers: unknown): number | undefined { if (!headers) { return undefined; } let raw: string | null | undefined; if (typeof (headers as { get?: unknown }).get === "function") { raw = (headers as { get(k: string): string | null }).get("retry-after"); } else if (typeof headers === "object") { raw = (headers as Record)["retry-after"]; } if (typeof raw === "string") { return parseRetryAfterMs(raw); } return undefined; } return { DEFAULT_MAX_RETRIES, DEFAULT_BASE_DELAY_MS, computeRetryDelay, parseRetryAfterMs, getHttpRetryDelay, isRetryableStatus, isRetryableNetworkError, extractRetryAfterMs, sleep: sleepSpy, abortableSleep: () => Promise.resolve(), }; }); import { RetryProvider } from "../providers/retry.js"; import { createStreamTimeout } from "../providers/stream-timeout.js"; import { ContextOverflowError, type Message, type Provider, type ProviderEvent, type ProviderResponse, type SendMessageOptions, } from "../providers/types.js"; import { ProviderError } from "../util/errors.js"; import { DEFAULT_MAX_RETRIES } from "../util/retry.js"; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- const MESSAGES: Message[] = [ { role: "user", content: [{ type: "text", text: "Hello" }] }, ]; function successResponse( overrides?: Partial, ): ProviderResponse { return { content: [{ type: "text", text: "ok" }], model: "test-model", usage: { inputTokens: 10, outputTokens: 5 }, stopReason: "end_turn", ...overrides, }; } /** Provider that fails N times then succeeds. */ function makeFlaky( failCount: number, error: Error, name = "flaky", ): Provider & { calls: number } { const p = { name, calls: 0, async sendMessage(): Promise { p.calls++; if (p.calls <= failCount) { throw error; } return successResponse(); }, }; return p; } /** Provider that always fails. */ function makeFailing( error: Error, name = "failing", ): Provider & { calls: number } { const p = { name, calls: 0, async sendMessage(): Promise { p.calls++; throw error; }, }; return p; } // --------------------------------------------------------------------------- // RetryProvider — rate limit backoff // --------------------------------------------------------------------------- describe("RetryProvider — rate limit backoff", () => { test("retries on 429 and succeeds after transient rate limit", async () => { const inner = makeFlaky(2, new ProviderError("rate limited", "test", 429)); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(result.content[0]).toMatchObject({ type: "text", text: "ok" }); // 2 failures + 1 success = 3 calls expect(inner.calls).toBe(3); }); test("throws after exhausting all retries on persistent 429", async () => { const inner = makeFailing(new ProviderError("rate limited", "test", 429)); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "rate limited", ); // 1 initial + DEFAULT_MAX_RETRIES retries expect(inner.calls).toBe(DEFAULT_MAX_RETRIES + 1); }); test("preserves ProviderError properties through retry exhaustion", async () => { const inner = makeFailing( new ProviderError("quota exceeded", "anthropic", 429), ); const provider = new RetryProvider(inner); try { await provider.sendMessage(MESSAGES); expect(true).toBe(false); // should not reach } catch (err) { expect(err).toBeInstanceOf(ProviderError); const pe = err as ProviderError; expect(pe.provider).toBe("anthropic"); expect(pe.statusCode).toBe(429); expect(pe.message).toBe("quota exceeded"); } }); test("attributes the credential selected for the failed request", async () => { const inner = makeFailing( new ProviderError("invalid key", "anthropic", 401), ); const provider = new RetryProvider(inner, { credentialSource: "byok", connectionName: "anthropic-personal", }); try { await provider.sendMessage(MESSAGES); expect(true).toBe(false); } catch (err) { expect(err).toBeInstanceOf(ProviderError); const providerError = err as ProviderError; expect(providerError.routeAttribution).toMatchObject({ credentialSource: "byok", connectionName: "anthropic-personal", }); } }); test("tags final error with retriesExhausted=true after retry loop gives up (JARVIS-513)", async () => { // Transient socket flap from Bun's native fetch: wrapped in a // ProviderError but still network-retryable via message pattern match. const inner = makeFailing( new ProviderError( "Anthropic request failed: The socket connection was closed unexpectedly", "anthropic", ), ); const provider = new RetryProvider(inner); try { await provider.sendMessage(MESSAGES); expect(true).toBe(false); } catch (err) { const flag = (err as Error & { retriesExhausted?: boolean }) .retriesExhausted; expect(flag).toBe(true); // Retry loop should have used all attempts before surrendering. expect(inner.calls).toBe(DEFAULT_MAX_RETRIES + 1); } }); test("does NOT tag retriesExhausted on non-retryable errors (no retry was attempted)", async () => { // ProviderError without statusCode and without a retryable pattern: the // retry loop short-circuits on the first attempt. No "exhaustion" // occurred, so the marker must stay unset. const inner = makeFailing(new ProviderError("model not found", "test")); const provider = new RetryProvider(inner); try { await provider.sendMessage(MESSAGES); expect(true).toBe(false); } catch (err) { const flag = (err as Error & { retriesExhausted?: boolean }) .retriesExhausted; expect(flag).toBeUndefined(); expect(inner.calls).toBe(1); } }); test("uses retryAfterMs from ProviderError when present", async () => { const error = new ProviderError("rate limited", "anthropic", 429, { retryAfterMs: 30_000, }); const inner = makeFlaky(1, error); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(result.content[0]).toMatchObject({ type: "text", text: "ok" }); expect(inner.calls).toBe(2); }); test("preserves retryAfterMs on ProviderError through retry exhaustion", async () => { const error = new ProviderError("rate limited", "anthropic", 429, { retryAfterMs: 60_000, }); const inner = makeFailing(error); const provider = new RetryProvider(inner); try { await provider.sendMessage(MESSAGES); expect(true).toBe(false); } catch (err) { expect(err).toBeInstanceOf(ProviderError); const pe = err as ProviderError; expect(pe.retryAfterMs).toBe(60_000); expect(pe.statusCode).toBe(429); } }); test("falls back to exponential backoff when retryAfterMs is absent", async () => { const error = new ProviderError("rate limited", "test", 429); expect(error.retryAfterMs).toBeUndefined(); const inner = makeFlaky(1, error); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(result.content[0]).toMatchObject({ type: "text", text: "ok" }); expect(inner.calls).toBe(2); }); test("caps retryAfterMs at MAX_RETRY_DELAY_MS", async () => { sleepSpy.mockClear(); const error = new ProviderError("rate limited", "anthropic", 429, { retryAfterMs: 3_600_000, // 1 hour - way too long }); const inner = makeFlaky(1, error); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(result.content[0]).toMatchObject({ type: "text", text: "ok" }); expect(inner.calls).toBe(2); // Verify sleep was called with the capped delay, not the original 3,600,000ms const sleepCalls = sleepSpy.mock.calls; const lastDelay = sleepCalls[sleepCalls.length - 1][0]; expect(lastDelay).toBe(60_000); }); }); describe("RetryProvider managed credential refresh", () => { test("reloads managed credentials once and keeps the refreshed provider", async () => { sleepSpy.mockClear(); const initial = makeFailing( new ProviderError("assistant key expired", "anthropic", 403, { reason: "invalid_credentials", }), "anthropic", ); const refreshed = makeFlaky( 0, new ProviderError("unused", "anthropic"), "anthropic", ); let refreshCalls = 0; const provider = new RetryProvider(initial, { credentialSource: "vellum-managed", connectionName: "vellum", refreshCredentialProvider: async () => { refreshCalls++; return refreshed; }, }); await expect(provider.sendMessage(MESSAGES)).resolves.toMatchObject({ model: "test-model", }); await expect(provider.sendMessage(MESSAGES)).resolves.toMatchObject({ model: "test-model", }); expect(initial.calls).toBe(1); expect(refreshed.calls).toBe(2); expect(refreshCalls).toBe(1); expect(sleepSpy).not.toHaveBeenCalled(); }); test("surfaces a managed auth failure when refreshed credentials are also rejected", async () => { const initial = makeFailing( new ProviderError("old key rejected", "gemini", 401), "gemini", ); const refreshed = makeFailing( new ProviderError("new key rejected", "gemini", 403), "gemini", ); let refreshCalls = 0; const provider = new RetryProvider(initial, { credentialSource: "vellum-managed", connectionName: "vellum", refreshCredentialProvider: async () => { refreshCalls++; return refreshed; }, }); try { await provider.sendMessage(MESSAGES); expect.unreachable(); } catch (error) { expect(error).toBeInstanceOf(ProviderError); const providerError = error as ProviderError; expect(providerError.message).toBe("new key rejected"); expect(providerError.routeAttribution).toMatchObject({ credentialSource: "vellum-managed", connectionName: "vellum", }); } expect(initial.calls).toBe(1); expect(refreshed.calls).toBe(1); expect(refreshCalls).toBe(1); }); test("does not reload credentials for a personal provider key rejection", async () => { const initial = makeFailing( new ProviderError("personal key rejected", "anthropic", 401), "anthropic", ); let refreshCalls = 0; const provider = new RetryProvider(initial, { credentialSource: "byok", refreshCredentialProvider: async () => { refreshCalls++; return null; }, }); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "personal key rejected", ); expect(initial.calls).toBe(1); expect(refreshCalls).toBe(0); }); test("does not reload managed credentials for a model restriction", async () => { const initial = makeFailing( new ProviderError("model unavailable", "anthropic", 403, { reason: "model_restricted", }), "anthropic", ); let refreshCalls = 0; const provider = new RetryProvider(initial, { credentialSource: "vellum-managed", refreshCredentialProvider: async () => { refreshCalls++; return null; }, }); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "model unavailable", ); expect(initial.calls).toBe(1); expect(refreshCalls).toBe(0); }); }); // --------------------------------------------------------------------------- // RetryProvider — ContextOverflowError short-circuit // --------------------------------------------------------------------------- describe("RetryProvider — ContextOverflowError short-circuit", () => { test("does NOT retry ContextOverflowError even when statusCode is 429", async () => { // Gemini/Vertex can surface context overflow as a 429 RESOURCE_EXHAUSTED. // Retrying is deterministic waste — the oversized prompt won't shrink. const error = new ContextOverflowError("prompt too long", "gemini", { statusCode: 429, actualTokens: 250_000, maxTokens: 200_000, }); const inner = makeFailing(error); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toBeInstanceOf( ContextOverflowError, ); // 1 call, no retries — must short-circuit before the 429 retry branch. expect(inner.calls).toBe(1); }); test("does NOT retry ContextOverflowError with default statusCode 400", async () => { // Anthropic / OpenAI-compatible providers surface overflow as 400. Same // deterministic-failure rule: never retry. const error = new ContextOverflowError("prompt is too long", "anthropic"); const inner = makeFailing(error); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toBeInstanceOf( ContextOverflowError, ); expect(inner.calls).toBe(1); }); test("still retries a plain ProviderError with statusCode 429 (non-overflow)", async () => { // Rate-limit 429s that are NOT overflow must continue to retry — this is // the regression guard for the short-circuit's scope. const inner = makeFailing( new ProviderError("rate limited", "anthropic", 429), ); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "rate limited", ); // 1 initial + DEFAULT_MAX_RETRIES retries expect(inner.calls).toBe(DEFAULT_MAX_RETRIES + 1); }); }); // --------------------------------------------------------------------------- // RetryProvider — server error retries // --------------------------------------------------------------------------- describe("RetryProvider — server error retries", () => { test("retries on 500 Internal Server Error", async () => { const inner = makeFlaky( 1, new ProviderError("internal error", "test", 500), ); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(result.stopReason).toBe("end_turn"); expect(inner.calls).toBe(2); }); test("retries on 502 Bad Gateway", async () => { const inner = makeFlaky(1, new ProviderError("bad gateway", "test", 502)); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(result.model).toBe("test-model"); }); test("retries on 503 Service Unavailable", async () => { const inner = makeFlaky(1, new ProviderError("unavailable", "test", 503)); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(result.content).toHaveLength(1); }); test("does not retry on 400 Bad Request", async () => { const inner = makeFailing(new ProviderError("bad request", "test", 400)); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow("bad request"); expect(inner.calls).toBe(1); }); test("does not retry on 401 Unauthorized", async () => { const inner = makeFailing(new ProviderError("unauthorized", "test", 401)); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "unauthorized", ); expect(inner.calls).toBe(1); }); test("does not retry on 403 Forbidden", async () => { const inner = makeFailing(new ProviderError("forbidden", "test", 403)); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow("forbidden"); expect(inner.calls).toBe(1); }); test("does not retry on 422 Unprocessable Entity", async () => { const inner = makeFailing(new ProviderError("invalid input", "test", 422)); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "invalid input", ); expect(inner.calls).toBe(1); }); }); // --------------------------------------------------------------------------- // RetryProvider — reason-driven retryability // --------------------------------------------------------------------------- describe("RetryProvider — reason-driven retryability", () => { for (const reason of [ "rate_limited", "overloaded", "server_error", ] as const) { test(`retries a ProviderError with reason=${reason}`, async () => { const inner = makeFlaky( 1, new ProviderError("transient", "openai", undefined, { reason }), ); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(result.stopReason).toBe("end_turn"); expect(inner.calls).toBe(2); }); } for (const reason of [ "invalid_credentials", "model_restricted", "context_overflow", ] as const) { test(`does NOT retry a ProviderError with reason=${reason}`, async () => { // A retryable-looking 429 status must not override a terminal reason. const inner = makeFailing( new ProviderError("terminal", "openai", 429, { reason }), ); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow("terminal"); expect(inner.calls).toBe(1); }); } test("reason=unknown falls through to the status fallback (429 retries)", async () => { const inner = makeFlaky( 1, new ProviderError("rate limited", "openai", 429, { reason: "unknown" }), ); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(result.stopReason).toBe("end_turn"); expect(inner.calls).toBe(2); }); }); // --------------------------------------------------------------------------- // RetryProvider — network error retries // --------------------------------------------------------------------------- describe("RetryProvider — network error retries", () => { test("retries on ECONNRESET", async () => { const err = new Error("connection reset"); (err as NodeJS.ErrnoException).code = "ECONNRESET"; const inner = makeFlaky(1, err); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(result.stopReason).toBe("end_turn"); }); test("retries on ECONNREFUSED", async () => { const err = new Error("connection refused"); (err as NodeJS.ErrnoException).code = "ECONNREFUSED"; const inner = makeFlaky(1, err); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(result.model).toBe("test-model"); }); test("retries on ETIMEDOUT", async () => { const err = new Error("timed out"); (err as NodeJS.ErrnoException).code = "ETIMEDOUT"; const inner = makeFlaky(1, err); const provider = new RetryProvider(inner); const _result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); }); test("retries on ECONNRESET in error cause chain", async () => { const cause = new Error("socket hangup"); (cause as NodeJS.ErrnoException).code = "ECONNRESET"; const outer = new Error("fetch failed", { cause }); const inner = makeFlaky(1, outer); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(result.content[0]).toMatchObject({ type: "text", text: "ok" }); }); test("retries on Bun 'socket connection was closed unexpectedly' (ProviderError wrapping)", async () => { const inner = makeFlaky( 1, new ProviderError( "Anthropic request failed: The socket connection was closed unexpectedly. For more information, pass `verbose: true` in the second argument to fetch()", "anthropic", ), ); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(result.stopReason).toBe("end_turn"); }); test("retries on 'socket connection was closed unexpectedly' in error cause", async () => { const cause = new Error( "The socket connection was closed unexpectedly. For more information, pass `verbose: true` in the second argument to fetch()", ); const outer = new Error("fetch failed", { cause }); const inner = makeFlaky(1, outer); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(result.content[0]).toMatchObject({ type: "text", text: "ok" }); }); test("does not retry on non-retryable errors", async () => { const inner = makeFailing(new Error("unexpected error")); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "unexpected error", ); expect(inner.calls).toBe(1); }); test("does not retry on ProviderError without status code (non-network)", async () => { // ProviderError without a statusCode and without a retryable network code const err = new ProviderError("model not found", "test"); const inner = makeFailing(err); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "model not found", ); expect(inner.calls).toBe(1); }); test("does NOT retry a ProviderError tagged with abortReason", async () => { // Defensive: if any future retryable pattern matches an error carrying // a daemon/user-initiated abortReason, the abortReason guard in // providers/retry.ts:isRetryableError must still short-circuit it. const inner = makeFailing( new ProviderError( "Anthropic request failed: socket closed unexpectedly", "anthropic", undefined, { abortReason: { kind: "user_cancel", source: "test" } }, ), ); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "socket closed", ); expect(inner.calls).toBe(1); }); test("does NOT retry 'Anthropic stream timed out' (inner streamTimeoutMs fired)", async () => { const inner = makeFailing( new ProviderError( "Anthropic API error: Anthropic stream timed out after 1800s (inner streamTimeoutMs)", "anthropic", ), ); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "stream timed out", ); expect(inner.calls).toBe(1); }); }); // --------------------------------------------------------------------------- // RetryProvider — streaming corruption retries // --------------------------------------------------------------------------- describe("RetryProvider — streaming corruption retries", () => { test("retries on 'Unexpected event order' (message_start before message_stop)", async () => { const inner = makeFlaky( 1, new ProviderError( 'Anthropic request failed: Unexpected event order, got message_start before receiving "message_stop"', "anthropic", ), ); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(result.stopReason).toBe("end_turn"); expect(inner.calls).toBe(2); }); test("retries on 'Unexpected event order' (event before message_start)", async () => { const inner = makeFlaky( 1, new ProviderError( 'Anthropic request failed: Unexpected event order, got content_block_start before "message_start"', "anthropic", ), ); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(result.model).toBe("test-model"); }); test("retries on 'stream ended without producing'", async () => { const inner = makeFlaky( 1, new ProviderError( "Anthropic request failed: stream ended without producing a Message with role=assistant", "anthropic", ), ); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(result.content).toHaveLength(1); }); test("retries on 'request ended without sending any chunks'", async () => { const inner = makeFlaky( 1, new ProviderError( "Anthropic request failed: request ended without sending any chunks", "anthropic", ), ); const provider = new RetryProvider(inner); await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); }); test("retries on 'Unable to parse tool parameter JSON' (invalid tool-args JSON in stream)", async () => { const inner = makeFlaky( 1, new ProviderError( 'Anthropic request failed: Unable to parse tool parameter JSON from model. Please retry your request or adjust your prompt. Error: SyntaxError: JSON Parse error: Unterminated string. JSON: {"path": "/workspace/config.json", "content', "anthropic", ), ); const provider = new RetryProvider(inner); const result = await provider.sendMessage(MESSAGES); expect(result.stopReason).toBe("end_turn"); expect(inner.calls).toBe(2); }); test("throws after exhausting retries on persistent stream corruption", async () => { const inner = makeFailing( new ProviderError( 'Anthropic request failed: Unexpected event order, got message_start before receiving "message_stop"', "anthropic", ), ); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "Unexpected event order", ); expect(inner.calls).toBe(DEFAULT_MAX_RETRIES + 1); }); /** Provider that fails N times then succeeds, recording each call's messages. */ function makeCapturingFlaky( failCount: number, error: Error, ): Provider & { calls: number; seen: Message[][] } { const p = { name: "capturing", calls: 0, seen: [] as Message[][], async sendMessage(messages: Message[]): Promise { p.calls++; p.seen.push(messages); if (p.calls <= failCount) { throw error; } return successResponse(); }, }; return p; } const PARSE_ERROR = new ProviderError( "Anthropic request failed: Unable to parse tool parameter JSON from " + "model. Please retry your request or adjust your prompt. Error: " + "SyntaxError: JSON Parse error: Expected ']'. JSON: {\"content\": [Jul", "anthropic", ); test("tool-JSON parse retries append a corrective note to the latest user message", async () => { const inner = makeCapturingFlaky(1, PARSE_ERROR); const provider = new RetryProvider(inner); await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); // First attempt sends the caller's messages untouched. expect(inner.seen[0]).toBe(MESSAGES); // Retry appends exactly one trailing text block to the last user message. const retried = inner.seen[1]!; const lastMessage = retried[retried.length - 1]!; expect(lastMessage.role).toBe("user"); expect(lastMessage.content).toHaveLength(2); const hint = lastMessage.content[1] as { type: string; text: string }; expect(hint.type).toBe("text"); expect(hint.text).toContain("strict JSON"); // The caller's array is never mutated. expect(MESSAGES[MESSAGES.length - 1]!.content).toHaveLength(1); }); test("the corrective note is applied once across repeated parse failures", async () => { const inner = makeCapturingFlaky(3, PARSE_ERROR); const provider = new RetryProvider(inner); await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(4); for (const attempt of inner.seen.slice(1)) { const lastMessage = attempt[attempt.length - 1]!; expect(lastMessage.content).toHaveLength(2); } }); test("other stream-corruption retries resend the messages untouched", async () => { const inner = makeCapturingFlaky( 1, new ProviderError( "Anthropic request failed: request ended without sending any chunks", "anthropic", ), ); const provider = new RetryProvider(inner); await provider.sendMessage(MESSAGES); expect(inner.calls).toBe(2); expect(inner.seen[1]).toBe(MESSAGES); }); test("does not retry non-stream ProviderError without status code", async () => { const inner = makeFailing( new ProviderError("model not found", "anthropic"), ); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "model not found", ); expect(inner.calls).toBe(1); }); test("does not treat stream pattern as retryable when ProviderError has a status code", async () => { // A 400 error that happens to contain "Unexpected event order" should NOT be retried const inner = makeFailing( new ProviderError( "Unexpected event order in request payload", "anthropic", 400, ), ); const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toThrow( "Unexpected event order", ); expect(inner.calls).toBe(1); }); }); // --------------------------------------------------------------------------- // RetryProvider — streaming + options passthrough // --------------------------------------------------------------------------- describe("RetryProvider — streaming response handling", () => { test("passes onEvent callback through to inner provider", async () => { const events: ProviderEvent[] = []; const inner: Provider = { name: "streaming-mock", async sendMessage(_m: Message[], options?: SendMessageOptions) { options?.onEvent?.({ type: "text_delta", text: "hello " }); options?.onEvent?.({ type: "text_delta", text: "world" }); return successResponse(); }, }; const provider = new RetryProvider(inner); await provider.sendMessage(MESSAGES, { onEvent: (e) => events.push(e), }); expect(events).toHaveLength(2); expect(events[0]).toMatchObject({ type: "text_delta", text: "hello " }); expect(events[1]).toMatchObject({ type: "text_delta", text: "world" }); }); test("passes signal through to inner provider", async () => { let receivedSignal: AbortSignal | undefined; const inner: Provider = { name: "signal-mock", async sendMessage(_m: Message[], options?: SendMessageOptions) { receivedSignal = options?.signal; return successResponse(); }, }; const provider = new RetryProvider(inner); const controller = new AbortController(); await provider.sendMessage(MESSAGES, { signal: controller.signal, }); expect(receivedSignal).toBe(controller.signal); }); test("retries overloaded_error with undefined statusCode (mid-stream SSE)", async () => { let callCount = 0; const inner: Provider = { name: "retry-overloaded-undefined", async sendMessage() { callCount++; if (callCount <= 1) { throw new ProviderError( 'Anthropic API error: {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}', "anthropic", undefined, ); } return successResponse(); }, }; const provider = new RetryProvider(inner); await provider.sendMessage(MESSAGES); expect(callCount).toBe(2); }); test("retries transport-aborted stream (Anthropic 'Request was aborted' with no abortReason)", async () => { let callCount = 0; const inner: Provider = { name: "retry-transport-abort", async sendMessage() { callCount++; if (callCount <= 1) { // Mirrors the ProviderError shape produced by the catch-site in // providers/anthropic/client.ts when the SDK reports // ``Anthropic.APIError(status === undefined, message: "Request // was aborted.")`` and the daemon's AbortController was NOT the // cause (i.e. abortReason is undefined). Empirically the #1 // daemon error by a factor of 5x — 1,344 events in 4d on the // SSE chat path, all of which used to surface as a 45s blank // screen on the web client via LUM-1431. throw new ProviderError( "Anthropic API error: Request was aborted.", "anthropic", undefined, ); } return successResponse(); }, }; const provider = new RetryProvider(inner); await provider.sendMessage(MESSAGES); expect(callCount).toBe(2); }); test("does NOT retry caller-aborted stream (abortReason set short-circuits retry)", async () => { let callCount = 0; const abortError = new ProviderError( "Anthropic API error: Request was aborted.", "anthropic", undefined, // Tagging abortReason exactly matches what the catch-site does when // signal.aborted was true at the moment of failure — i.e. the // daemon (or the user) cancelled the request, not the transport. // The retry layer must respect this and surface the error // immediately without consuming retry budget. { abortReason: "user-cancelled" }, ); const inner: Provider = { name: "caller-abort", async sendMessage() { callCount++; throw abortError; }, }; const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toBe(abortError); expect(callCount).toBe(1); }); test("does NOT retry inner-timeout stream (deterministic 30min deadline failure)", async () => { let callCount = 0; // When the inner streamTimeoutMs fires, the catch-site rewrites the // message to "Anthropic stream timed out after Xs (inner // streamTimeoutMs)" instead of "Request was aborted." That rewrite is // what allows this branch to bypass the transport-abort retry — // retrying a 30min-deadline failure would almost certainly hit the // same deadline on the next attempt and waste retry budget. const innerTimeoutError = new ProviderError( "Anthropic API error: Anthropic stream timed out after 1800s (inner streamTimeoutMs)", "anthropic", undefined, ); const inner: Provider = { name: "inner-timeout", async sendMessage() { callCount++; throw innerTimeoutError; }, }; const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toBe( innerTimeoutError, ); expect(callCount).toBe(1); }); test("does NOT retry OpenAI/Gemini-shaped 'Request was aborted' (no inner-timeout rewrite at those catch-sites)", async () => { // The OpenAI chat-completions, OpenAI responses, and Gemini catch-sites // format their errors as `" API error (): Request // was aborted."` (the OpenAI catch-sites render a missing status as // `(unknown status)`; the Anthropic catch-site intentionally omits the // parenthetical) and — unlike the Anthropic // catch-site — they do NOT rewrite their inner-streamTimeoutMs // deadline failures. A provider-agnostic transport-abort predicate // would burn three retries on what is by construction a deterministic // 30-minute deadline failure that will fire again on every attempt. // Scoping the predicate to the Anthropic message prefix avoids that // wasted retry budget for non-Anthropic providers until their // catch-sites grow the same `innerTimeoutFired` distinction. const openaiAbortError = new ProviderError( "OpenAI API error (unknown status): Request was aborted.", "openai", undefined, ); let callCount = 0; const inner: Provider = { name: "openai-aborted-stream", async sendMessage() { callCount++; throw openaiAbortError; }, }; const provider = new RetryProvider(inner); await expect(provider.sendMessage(MESSAGES)).rejects.toBe(openaiAbortError); expect(callCount).toBe(1); }); test("events accumulate across retries (each attempt delivers events independently)", async () => { let callCount = 0; const inner: Provider = { name: "retry-stream", async sendMessage(_m: Message[], options?: SendMessageOptions) { callCount++; options?.onEvent?.({ type: "text_delta", text: `attempt${callCount} `, }); if (callCount <= 1) { throw new ProviderError("overloaded", "test", 529); } return successResponse(); }, }; const provider = new RetryProvider(inner); const events: ProviderEvent[] = []; await provider.sendMessage(MESSAGES, { onEvent: (e) => events.push(e), }); // Events from both attempts are delivered expect(events).toHaveLength(2); expect(events[0]).toMatchObject({ type: "text_delta", text: "attempt1 " }); expect(events[1]).toMatchObject({ type: "text_delta", text: "attempt2 " }); }); }); // --------------------------------------------------------------------------- // createStreamTimeout — edge cases // --------------------------------------------------------------------------- describe("createStreamTimeout — edge cases", () => { test("propagates already-aborted external signal immediately", () => { const external = new AbortController(); external.abort(new Error("already cancelled")); const { signal, cleanup } = createStreamTimeout(60_000, external.signal); expect(signal.aborted).toBe(true); expect(signal.reason).toBeInstanceOf(Error); expect((signal.reason as Error).message).toBe("already cancelled"); cleanup(); }); test("cleanup prevents timeout from firing", async () => { const { signal, cleanup } = createStreamTimeout(50); cleanup(); await new Promise((r) => setTimeout(r, 100)); expect(signal.aborted).toBe(false); }); test("cleanup removes external signal listener", () => { const external = new AbortController(); const { signal, cleanup } = createStreamTimeout(60_000, external.signal); cleanup(); // Aborting external after cleanup should NOT propagate external.abort(new Error("late abort")); expect(signal.aborted).toBe(false); }); test("timeout error message includes duration", async () => { const { signal, cleanup } = createStreamTimeout(100); await new Promise((resolve) => { signal.addEventListener("abort", () => resolve(), { once: true }); }); expect(signal.reason).toBeInstanceOf(Error); expect((signal.reason as Error).message).toContain("0.1s"); cleanup(); }); });