/** * Agentic QE v3 - LLM Provider Manager * ADR-011: LLM Provider System for Quality Engineering * * Manages multiple LLM providers with: * - Load balancing (round-robin, least-cost, least-latency, random) * - Automatic failover * - Circuit breaker integration * - Response caching * - Cost tracking */ import { LLMProvider, LLMProviderType, ProviderManagerConfig, ProviderSelection, ProviderMetrics, Message, GenerateOptions, EmbedOptions, CompleteOptions, LLMResponse, EmbeddingResponse, CompletionResponse, HealthCheckResult, CircuitBreakerConfig, LLMCacheConfig, createLLMError, } from './interfaces'; import { CircuitBreaker, CircuitBreakerManager } from './circuit-breaker'; import { LLMResponseCache } from './cache'; import { CostTracker, getGlobalCostTracker } from './cost-tracker'; import { ClaudeProvider } from './providers/claude'; import { OpenAIProvider } from './providers/openai'; import { OllamaProvider } from './providers/ollama'; /** * Default provider manager configuration */ export const DEFAULT_PROVIDER_MANAGER_CONFIG: ProviderManagerConfig = { primary: 'claude', fallbacks: ['openai', 'ollama'], loadBalancing: 'round-robin', providers: {}, global: { enableCostTracking: true, enableMetrics: true, }, }; /** * Metrics tracking for providers */ interface ProviderMetricsInternal { totalRequests: number; successCount: number; failureCount: number; latencies: number[]; totalCost: number; totalTokens: number; } /** * LLM Provider Manager * Coordinates multiple LLM providers with load balancing and failover */ export class ProviderManager { private providers: Map = new Map(); private circuitBreakers: CircuitBreakerManager; private cache: LLMResponseCache; private costTracker: CostTracker; private config: ProviderManagerConfig; private metrics: Map = new Map(); private roundRobinIndex: number = 0; private initialized: boolean = false; constructor( config: Partial = {}, options?: { circuitBreakerConfig?: Partial; cacheConfig?: Partial; costTracker?: CostTracker; } ) { this.config = { ...DEFAULT_PROVIDER_MANAGER_CONFIG, ...config }; this.circuitBreakers = new CircuitBreakerManager(options?.circuitBreakerConfig); this.cache = new LLMResponseCache(options?.cacheConfig); this.costTracker = options?.costTracker ?? getGlobalCostTracker(); } /** * Initialize the provider manager */ async initialize(): Promise { if (this.initialized) { return; } // Create provider instances based on config await this.createProviders(); // Initialize metrics for each provider for (const type of this.providers.keys()) { this.initializeMetrics(type); } this.initialized = true; } /** * Generate text using the best available provider */ async generate( input: string | Message[], options?: GenerateOptions ): Promise { await this.ensureInitialized(); // Check cache first if (!options?.skipCache) { const inputStr = typeof input === 'string' ? input : JSON.stringify(input); const cached = this.cache.getGeneration(inputStr, { model: options?.model, temperature: options?.temperature, maxTokens: options?.maxTokens, systemPrompt: options?.systemPrompt, }); if (cached) { return { ...cached, cached: true }; } } // Select provider and execute with failover const response = await this.executeWithFailover( 'generate', async (provider) => provider.generate(input, options), options?.preferredProvider ); // Cache the response if (!options?.skipCache) { const inputStr = typeof input === 'string' ? input : JSON.stringify(input); this.cache.setGeneration(inputStr, response, { model: options?.model, temperature: options?.temperature, maxTokens: options?.maxTokens, systemPrompt: options?.systemPrompt, }); } // Track cost this.costTracker.recordUsage( response.provider, response.model, response.usage, response.requestId ); return response; } /** * Generate embedding using the best available provider */ async embed(text: string, options?: EmbedOptions): Promise { await this.ensureInitialized(); // Check cache first if (!options?.skipCache) { const cached = this.cache.getEmbedding(text, { model: options?.model }); if (cached) { return { ...cached, cached: true }; } } // For embeddings, prefer providers that support them const embeddingProviders: LLMProviderType[] = ['openai', 'ollama']; const response = await this.executeWithFailover( 'embed', async (provider) => provider.embed(text, options), embeddingProviders.find((p) => this.providers.has(p)) ); // Cache the response if (!options?.skipCache) { this.cache.setEmbedding(text, response, { model: options?.model }); } return response; } /** * Complete text using the best available provider */ async complete( prompt: string, options?: CompleteOptions ): Promise { await this.ensureInitialized(); // Check cache first if (!options?.skipCache) { const cached = this.cache.getCompletion(prompt, { model: options?.model, temperature: options?.temperature, maxTokens: options?.maxTokens, }); if (cached) { return { ...cached, cached: true }; } } const response = await this.executeWithFailover( 'complete', async (provider) => provider.complete(prompt, options) ); // Cache the response if (!options?.skipCache) { this.cache.setCompletion(prompt, response, { model: options?.model, temperature: options?.temperature, maxTokens: options?.maxTokens, }); } return response; } /** * Health check all providers */ async healthCheck(): Promise> { await this.ensureInitialized(); const results: Partial> = {}; for (const [type, provider] of this.providers) { results[type] = await provider.healthCheck(); } return results as Record; } /** * Get a specific provider */ getProvider(type: LLMProviderType): LLMProvider | undefined { return this.providers.get(type); } /** * Get all available providers */ getAvailableProviders(): LLMProviderType[] { const available: LLMProviderType[] = []; for (const [type, _provider] of this.providers) { const breaker = this.circuitBreakers.getBreaker(type); if (breaker.canExecute()) { available.push(type); } } return available; } /** * Get metrics for all providers */ getMetrics(): Record { const result: Partial> = {}; for (const [type, internal] of this.metrics) { const breaker = this.circuitBreakers.getBreaker(type); const latencies = internal.latencies.slice(-100); // Last 100 for percentiles result[type] = { provider: type, totalRequests: internal.totalRequests, successCount: internal.successCount, failureCount: internal.failureCount, avgLatencyMs: this.calculateAverage(latencies), p95LatencyMs: this.calculatePercentile(latencies, 95), p99LatencyMs: this.calculatePercentile(latencies, 99), totalCost: internal.totalCost, totalTokens: internal.totalTokens, circuitState: breaker.getState(), }; } return result as Record; } /** * Get cache statistics */ getCacheStats() { return this.cache.getStats(); } /** * Get cost summary */ getCostSummary(period: 'hour' | 'day' | 'week' | 'month' | 'all' = 'day') { return this.costTracker.getSummary(period); } /** * Clear cache */ clearCache(): void { this.cache.clear(); } /** * Reset circuit breakers */ resetCircuitBreakers(): void { this.circuitBreakers.resetAll(); } /** * Dispose all resources */ async dispose(): Promise { for (const provider of this.providers.values()) { await provider.dispose(); } this.providers.clear(); this.metrics.clear(); this.cache.clear(); this.initialized = false; } /** * Create provider instances based on configuration */ private async createProviders(): Promise { // Always try to create all providers that are configured or in the fallback list const providerTypes = new Set([ this.config.primary, ...this.config.fallbacks, ]); for (const type of providerTypes) { try { const provider = this.createProvider(type); this.providers.set(type, provider); } catch (error) { console.warn( `Failed to create ${type} provider: ${error instanceof Error ? error.message : 'Unknown'}` ); } } if (this.providers.size === 0) { throw createLLMError( 'No LLM providers could be initialized', 'PROVIDER_UNAVAILABLE', { retryable: false } ); } } /** * Create a single provider instance */ private createProvider(type: LLMProviderType): LLMProvider { switch (type) { case 'claude': return new ClaudeProvider(this.config.providers.claude); case 'openai': return new OpenAIProvider(this.config.providers.openai); case 'ollama': return new OllamaProvider(this.config.providers.ollama); default: throw new Error(`Unknown provider type: ${type}`); } } /** * Initialize metrics for a provider */ private initializeMetrics(type: LLMProviderType): void { this.metrics.set(type, { totalRequests: 0, successCount: 0, failureCount: 0, latencies: [], totalCost: 0, totalTokens: 0, }); } /** * Execute a provider operation with failover */ private async executeWithFailover( operation: string, fn: (provider: LLMProvider) => Promise, preferredProvider?: LLMProviderType ): Promise { const selection = this.selectProvider(preferredProvider); const triedProviders = new Set(); let lastError: Error | undefined; // Try the selected provider first, then fallbacks const providersToTry = [ selection.provider.type, ...this.config.fallbacks.filter((p) => p !== selection.provider.type), ]; for (const providerType of providersToTry) { if (triedProviders.has(providerType)) { continue; } triedProviders.add(providerType); const provider = this.providers.get(providerType); if (!provider) { continue; } const breaker = this.circuitBreakers.getBreaker(providerType); if (!breaker.canExecute()) { continue; } try { const start = Date.now(); const result = await breaker.execute(() => fn(provider)); const latencyMs = Date.now() - start; // Update metrics this.recordSuccess(providerType, latencyMs, result); return result; } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); // Update metrics this.recordFailure(providerType, lastError); // Check if error is retryable if ('retryable' in lastError && !(lastError as { retryable: boolean }).retryable) { throw lastError; } } } throw createLLMError( `All providers failed for ${operation}: ${lastError?.message ?? 'Unknown error'}`, 'PROVIDER_UNAVAILABLE', { retryable: false, cause: lastError } ); } /** * Select the best provider based on load balancing strategy */ private selectProvider(preferred?: LLMProviderType): ProviderSelection { // If preferred provider is specified and available, use it if (preferred) { const provider = this.providers.get(preferred); const breaker = this.circuitBreakers.getBreaker(preferred); if (provider && breaker.canExecute()) { return { provider, reason: 'primary' }; } } const availableProviders = this.getAvailableProviders(); if (availableProviders.length === 0) { // Fall back to primary even if circuit is open const primary = this.providers.get(this.config.primary); if (primary) { return { provider: primary, reason: 'fallback' }; } throw createLLMError( 'No providers available', 'PROVIDER_UNAVAILABLE', { retryable: false } ); } // Select based on load balancing strategy switch (this.config.loadBalancing) { case 'round-robin': return this.selectRoundRobin(availableProviders); case 'least-cost': return this.selectLeastCost(availableProviders); case 'least-latency': return this.selectLeastLatency(availableProviders); case 'random': return this.selectRandom(availableProviders); default: return this.selectRoundRobin(availableProviders); } } /** * Round-robin selection */ private selectRoundRobin(available: LLMProviderType[]): ProviderSelection { this.roundRobinIndex = (this.roundRobinIndex + 1) % available.length; const type = available[this.roundRobinIndex]; const provider = this.providers.get(type)!; return { provider, reason: 'load-balance' }; } /** * Select provider with least cost */ private selectLeastCost(available: LLMProviderType[]): ProviderSelection { let lowestCost = Infinity; let selectedType: LLMProviderType = available[0]; for (const type of available) { const provider = this.providers.get(type)!; const { input, output } = provider.getCostPerToken(); const cost = input + output; if (cost < lowestCost) { lowestCost = cost; selectedType = type; } } return { provider: this.providers.get(selectedType)!, reason: 'cost-optimization', metadata: { estimatedCost: lowestCost }, }; } /** * Select provider with least average latency */ private selectLeastLatency(available: LLMProviderType[]): ProviderSelection { let lowestLatency = Infinity; let selectedType: LLMProviderType = available[0]; for (const type of available) { const metrics = this.metrics.get(type); if (metrics && metrics.latencies.length > 0) { const avgLatency = this.calculateAverage(metrics.latencies); if (avgLatency < lowestLatency) { lowestLatency = avgLatency; selectedType = type; } } } return { provider: this.providers.get(selectedType)!, reason: 'latency-optimization', metadata: { avgLatency: lowestLatency }, }; } /** * Random selection */ private selectRandom(available: LLMProviderType[]): ProviderSelection { const index = Math.floor(Math.random() * available.length); const type = available[index]; return { provider: this.providers.get(type)!, reason: 'load-balance' }; } /** * Record a successful operation */ private recordSuccess(type: LLMProviderType, latencyMs: number, result: unknown): void { const metrics = this.metrics.get(type); if (!metrics) return; metrics.totalRequests++; metrics.successCount++; metrics.latencies.push(latencyMs); // Keep only last 1000 latencies if (metrics.latencies.length > 1000) { metrics.latencies = metrics.latencies.slice(-1000); } // Extract cost and tokens if available if (result && typeof result === 'object') { const r = result as { cost?: { totalCost?: number }; usage?: { totalTokens?: number } }; if (r.cost?.totalCost) { metrics.totalCost += r.cost.totalCost; } if (r.usage?.totalTokens) { metrics.totalTokens += r.usage.totalTokens; } } } /** * Record a failed operation */ private recordFailure(type: LLMProviderType, _error: Error): void { const metrics = this.metrics.get(type); if (!metrics) return; metrics.totalRequests++; metrics.failureCount++; } /** * Ensure the manager is initialized */ private async ensureInitialized(): Promise { if (!this.initialized) { await this.initialize(); } } /** * Calculate average of numbers */ private calculateAverage(numbers: number[]): number { if (numbers.length === 0) return 0; return numbers.reduce((a, b) => a + b, 0) / numbers.length; } /** * Calculate percentile of numbers */ private calculatePercentile(numbers: number[], percentile: number): number { if (numbers.length === 0) return 0; const sorted = [...numbers].sort((a, b) => a - b); const index = Math.ceil((percentile / 100) * sorted.length) - 1; return sorted[Math.max(0, index)]; } } /** * Create a pre-configured provider manager */ export function createProviderManager( config?: Partial ): ProviderManager { return new ProviderManager(config); } /** * Create a provider manager with default settings for QE */ export function createQEProviderManager(): ProviderManager { return new ProviderManager({ primary: 'claude', fallbacks: ['openai', 'ollama'], loadBalancing: 'least-cost', providers: { claude: { model: 'claude-sonnet-4-20250514', maxTokens: 8192, temperature: 0.3, // Lower for QE tasks }, openai: { model: 'gpt-4o', maxTokens: 8192, temperature: 0.3, }, ollama: { model: 'llama3.1', maxTokens: 4096, temperature: 0.3, }, }, global: { enableCostTracking: true, enableMetrics: true, maxCostPerDay: 100, // $100/day limit }, }); }