/** * @license * Copyright 2025 Vybestack LLC * SPDX-License-Identifier: Apache-2.0 * * @plan PLAN-20260128issue808 * RetryOrchestrator - Centralized retry and bucket failover management * * This wrapper implements the "external retry orchestrator" pattern, * moving ALL retry logic out of individual providers into a single, * consistent implementation that handles: * * 1. Exponential backoff with jitter * 2. OAuth bucket failover * 3. Circuit breaker pattern (optional) * 4. Throttle wait time tracking * 5. Abort signal propagation * * Architecture: * - Providers throw immediately on errors (fast-fail) * - RetryOrchestrator handles all retry/backoff/failover logic * - Works with BucketFailoverHandler from config * - Respects ephemeral settings (retries, retrywait) */ import { type IProvider, type GenerateChatOptions, type ProviderToolset } from './IProvider.js'; import type { IModel } from './IModel.js'; import type { IContent } from '../services/history/IContent.js'; export interface RetryOrchestratorConfig { /** Maximum retry attempts (default: 6) */ maxAttempts?: number; /** Initial delay in ms before first retry (default: 5000) */ initialDelayMs?: number; /** Maximum delay in ms between retries (default: 30000) */ maxDelayMs?: number; /** Enable circuit breaker pattern (default: false) */ circuitBreakerEnabled?: boolean; /** Number of failures before opening circuit (default: 3) */ circuitBreakerFailureThreshold?: number; /** Time window for counting failures in ms (default: 60000) */ circuitBreakerFailureWindowMs?: number; /** Time to wait before testing recovery in ms (default: 30000) */ circuitBreakerRecoveryTimeoutMs?: number; /** Timeout for first chunk in streaming mode in ms (optional) */ streamingTimeoutMs?: number; /** Callback to track throttle wait time for metrics */ trackThrottleWaitTime?: (waitTimeMs: number) => void; } export interface CircuitBreakerState { state: 'closed' | 'open' | 'half-open'; failures: Array<{ timestamp: number; error: Error; }>; openedAt?: number; lastAttempt?: number; } /** * RetryOrchestrator wraps a provider to add centralized retry, backoff, * and bucket failover logic. This enables the "fast-fail" pattern where * providers throw immediately on errors and the orchestrator handles retries. */ export declare class RetryOrchestrator implements IProvider { readonly name: string; readonly wrappedProvider: IProvider; private readonly logger; private readonly config; constructor(provider: IProvider, config?: RetryOrchestratorConfig); /** * Check if the wrapped provider is a LoadBalancingProvider * LoadBalancingProvider has its own retry/failover logic, so we should * pass through without adding retry orchestration */ private isLoadBalancer; getModels(): Promise; getDefaultModel(): string; getCurrentModel?(): string; getToolFormat?(): string; isPaidMode?(): boolean; getServerTools(): string[]; invokeServerTool(toolName: string, params: unknown, config?: unknown, signal?: AbortSignal): Promise; getModelParams?(): Record | undefined; clearAuthCache?(): void; clearAuth?(): void; /** * Main method with retry orchestration logic * Supports both overloaded signatures for backward compatibility */ generateChatCompletion(optionsOrContents: GenerateChatOptions | IContent[], tools?: ProviderToolset, signal?: AbortSignal): AsyncIterableIterator; /** * Core retry orchestration logic */ private generateChatCompletionWithRetry; /** * Wraps an async generator with a timeout for the first chunk */ private streamWithTimeout; /** * Determines if an error should trigger a retry */ private shouldRetryError; /** * Gets the delay duration for a retry, respecting Retry-After header */ private getDelayDuration; /** * Extracts Retry-After delay from error headers */ private getRetryAfterDelayMs; /** * Checks if error has a Retry-After header */ private hasRetryAfterHeader; /** * Gets the bucket failover handler from options */ private getBucketFailoverHandler; /** * Creates an AllBucketsExhaustedError with failure reasons * @plan PLAN-20260223-ISSUE1598.P16 * @requirement REQ-1598-IC09 */ private createAllBucketsExhaustedError; }