/** * Performance Analyzer - Real-time performance monitoring and bottleneck detection * * Monitors system performance, detects bottlenecks, analyzes trends, and provides * intelligent recommendations for optimization across the AI integration ecosystem. */ import * as os from 'os'; import { EventEmitter } from 'eventemitter3'; import { PerformanceConfig, PerformanceMetrics, Bottleneck, Task, OperationResult, } from '../types'; export class PerformanceAnalyzer extends EventEmitter { private config: PerformanceConfig; private metricsCollector: MetricsCollector; private bottleneckDetector: BottleneckDetector; private trendAnalyzer: TrendAnalyzer; private alertManager: AlertManager; private performanceHistory: PerformanceSnapshot[] = []; private monitoringTimer: NodeJS.Timeout | null = null; private isMonitoring: boolean = false; constructor(config: PerformanceConfig) { super(); this.config = config; this.metricsCollector = new MetricsCollector(); this.bottleneckDetector = new BottleneckDetector(config); this.trendAnalyzer = new TrendAnalyzer(); this.alertManager = new AlertManager(config); } async initialize(): Promise { try { // Initialize subsystems await this.metricsCollector.initialize(); await this.bottleneckDetector.initialize(); await this.trendAnalyzer.initialize(); await this.alertManager.initialize(); // Start monitoring if enabled if (this.config.metricsCollection) { await this.startMonitoring(); } return { success: true, message: 'Performance Analyzer initialized successfully', }; } catch (error) { return { success: false, message: `Performance Analyzer initialization failed: ${error.message}`, error: error, }; } } private async startMonitoring(): Promise { if (this.isMonitoring) return; this.isMonitoring = true; // Start periodic monitoring this.monitoringTimer = setInterval(async () => { await this.collectAndAnalyzeMetrics(); }, 5000); // Collect metrics every 5 seconds this.emit('monitoring-started'); } private async collectAndAnalyzeMetrics(): Promise { try { // Collect current metrics const metrics = await this.metricsCollector.collectMetrics(); // Create performance snapshot const snapshot: PerformanceSnapshot = { timestamp: new Date(), metrics: metrics, systemLoad: await this.calculateSystemLoad(metrics), healthScore: this.calculateHealthScore(metrics), }; // Store snapshot this.performanceHistory.push(snapshot); // Maintain history size if (this.performanceHistory.length > 1000) { this.performanceHistory.shift(); } // Analyze for bottlenecks if (this.config.bottleneckDetection) { const bottlenecks = await this.bottleneckDetector.detectBottlenecks(snapshot); if (bottlenecks.length > 0) { await this.handleBottlenecks(bottlenecks); } } // Analyze trends const trends = await this.trendAnalyzer.analyzeTrends( this.performanceHistory ); // Check alert conditions await this.alertManager.checkAlerts(snapshot, trends); this.emit('metrics-collected', snapshot); } catch (error) { console.error('Failed to collect metrics:', error); this.emit('metrics-error', error); } } private async calculateSystemLoad( metrics: PerformanceMetrics ): Promise { let load = 0; // CPU load (30%) load += (metrics.cpuUsage / 100) * 0.3; // Memory load (25%) load += (metrics.memoryUsage / 100) * 0.25; // Response time load (20%) const responseTimeScore = Math.min(metrics.responseTime / 5000, 1); // 5s max load += responseTimeScore * 0.2; // Error rate load (15%) load += metrics.errorRate * 0.15; // Queue length load (10%) const queueScore = Math.min(metrics.queueLength / 100, 1); // 100 max load += queueScore * 0.1; return Math.min(load, 1); } private calculateHealthScore(metrics: PerformanceMetrics): number { let health = 1.0; // Penalize high response times if (metrics.responseTime > this.config.alertThresholds.responseTime) { health -= 0.3; } // Penalize high error rates if (metrics.errorRate > this.config.alertThresholds.errorRate) { health -= 0.4; } // Penalize high memory usage if (metrics.memoryUsage > this.config.alertThresholds.memoryUsage) { health -= 0.2; } // Penalize low throughput if (metrics.throughput < 10) { health -= 0.1; } return Math.max(health, 0); } private async handleBottlenecks(bottlenecks: Bottleneck[]): Promise { for (const bottleneck of bottlenecks) { this.emit('bottleneck-detected', bottleneck); // Apply auto-optimization if enabled if (this.config.autoOptimization) { await this.applyOptimization(bottleneck); } } } private async applyOptimization(bottleneck: Bottleneck): Promise { try { const optimization = this.generateOptimizationStrategy(bottleneck); // Execute optimization actions for (const action of optimization.actions) { await this.executeOptimizationAction(action); } this.emit('optimization-applied', { bottleneck, optimization }); } catch (error) { console.error( `Failed to apply optimization for ${bottleneck.type}:`, error ); this.emit('optimization-failed', { bottleneck, error }); } } private generateOptimizationStrategy( bottleneck: Bottleneck ): OptimizationStrategy { const strategy: OptimizationStrategy = { target: bottleneck.type, severity: bottleneck.severity, actions: [], expectedImpact: 'medium', }; switch (bottleneck.type) { case 'memory': strategy.actions = [ 'trigger-memory-cleanup', 'enable-compression', 'optimize-cache-size', ]; strategy.expectedImpact = 'high'; break; case 'cpu': strategy.actions = [ 'distribute-workload', 'optimize-algorithms', 'enable-parallel-processing', ]; strategy.expectedImpact = 'medium'; break; case 'network': strategy.actions = [ 'enable-compression', 'optimize-batch-sizes', 'implement-caching', ]; strategy.expectedImpact = 'medium'; break; case 'agent': strategy.actions = [ 'spawn-additional-agents', 'redistribute-tasks', 'optimize-agent-selection', ]; strategy.expectedImpact = 'high'; break; case 'task-queue': strategy.actions = [ 'increase-parallelism', 'optimize-priority-scheduling', 'implement-load-balancing', ]; strategy.expectedImpact = 'high'; break; } return strategy; } private async executeOptimizationAction(action: string): Promise { // Simulate optimization action execution console.info(`Executing optimization action: ${action}`); switch (action) { case 'trigger-memory-cleanup': // Trigger garbage collection or memory cleanup if (global.gc) { global.gc(); } break; case 'enable-compression': // Enable data compression break; case 'distribute-workload': // Redistribute workload across agents break; default: console.info(`Action ${action} not implemented yet`); } // Simulate execution time await new Promise(resolve => setTimeout(resolve, 100)); } /** * Analyze task performance */ async analyzeTask(task: Task): Promise { const analysis: TaskAnalysis = { taskId: task.id, complexity: this.calculateTaskComplexity(task), estimatedResources: this.estimateResourceRequirements(task), riskFactors: this.identifyRiskFactors(task), recommendations: [], confidence: 0.8, }; // Generate recommendations based on analysis analysis.recommendations = this.generateTaskRecommendations(analysis); return analysis; } private calculateTaskComplexity(task: Task): number { let complexity = 1; // Base complexity from description complexity += Math.min(task.description.length / 100, 3); // Capability requirements complexity += task.requiredCapabilities.length * 0.5; // Priority weight const priorityWeights = { low: 1, medium: 1.5, high: 2, critical: 3 }; complexity *= priorityWeights[task.priority] || 1; // Type complexity const typeComplexity = { coding: 2.5, review: 1.5, testing: 2.0, analysis: 3.0, documentation: 1.2, deployment: 3.5, optimization: 4.0, }; complexity *= typeComplexity[task.type] || 1; return Math.round(complexity * 10) / 10; } private estimateResourceRequirements(task: Task): ResourceRequirements { const complexity = this.calculateTaskComplexity(task); return { cpuEstimate: Math.min(complexity * 10, 90), // Max 90% CPU memoryEstimate: Math.min(complexity * 50, 500), // Max 500MB timeEstimate: Math.ceil(complexity * 1000), // In milliseconds agentsRequired: Math.min(Math.ceil(complexity / 2), 6), // Max 6 agents }; } private identifyRiskFactors(task: Task): RiskFactor[] { const risks: RiskFactor[] = []; // High complexity risk const complexity = this.calculateTaskComplexity(task); if (complexity > 8) { risks.push({ type: 'high-complexity', severity: 'high', description: 'Task complexity may lead to longer execution times and higher resource usage', mitigation: 'Consider breaking down into smaller subtasks', }); } // Resource contention risk if (task.requiredCapabilities.length > 5) { risks.push({ type: 'resource-contention', severity: 'medium', description: 'Multiple capability requirements may cause agent contention', mitigation: 'Ensure adequate agent pool availability', }); } // Priority conflict risk if (task.priority === 'critical') { risks.push({ type: 'priority-conflict', severity: 'medium', description: 'Critical tasks may disrupt other ongoing work', mitigation: 'Plan resource allocation carefully', }); } return risks; } private generateTaskRecommendations(analysis: TaskAnalysis): string[] { const recommendations: string[] = []; // Complexity-based recommendations if (analysis.complexity > 6) { recommendations.push('Consider decomposing into smaller subtasks'); recommendations.push('Allocate additional monitoring resources'); } // Resource-based recommendations if (analysis.estimatedResources.memoryEstimate > 300) { recommendations.push('Enable memory optimization'); recommendations.push('Consider distributed execution'); } // Risk-based recommendations for (const risk of analysis.riskFactors) { recommendations.push(risk.mitigation); } // Performance-based recommendations const recentPerformance = this.getRecentPerformanceAverage(); if (recentPerformance.responseTime > 3000) { recommendations.push( 'System is under load - consider delaying non-critical tasks' ); } return [...new Set(recommendations)]; // Remove duplicates } private getRecentPerformanceAverage(): PerformanceMetrics { const recent = this.performanceHistory.slice(-10); // Last 10 snapshots if (recent.length === 0) { return { responseTime: 0, throughput: 0, errorRate: 0, memoryUsage: 0, cpuUsage: 0, activeConnections: 0, queueLength: 0, }; } const avg = recent.reduce( (sum, snapshot) => ({ responseTime: sum.responseTime + snapshot.metrics.responseTime, throughput: sum.throughput + snapshot.metrics.throughput, errorRate: sum.errorRate + snapshot.metrics.errorRate, memoryUsage: sum.memoryUsage + snapshot.metrics.memoryUsage, cpuUsage: sum.cpuUsage + snapshot.metrics.cpuUsage, activeConnections: sum.activeConnections + snapshot.metrics.activeConnections, queueLength: sum.queueLength + snapshot.metrics.queueLength, }), { responseTime: 0, throughput: 0, errorRate: 0, memoryUsage: 0, cpuUsage: 0, activeConnections: 0, queueLength: 0, } ); const count = recent.length; return { responseTime: avg.responseTime / count, throughput: avg.throughput / count, errorRate: avg.errorRate / count, memoryUsage: avg.memoryUsage / count, cpuUsage: avg.cpuUsage / count, activeConnections: avg.activeConnections / count, queueLength: avg.queueLength / count, }; } /** * Get comprehensive performance report */ async generatePerformanceReport(): Promise { const currentMetrics = await this.metricsCollector.collectMetrics(); const recentTrends = await this.trendAnalyzer.analyzeTrends( this.performanceHistory ); const activeBottlenecks = await this.bottleneckDetector.getCurrentBottlenecks(); return { timestamp: new Date(), currentMetrics, trends: recentTrends, bottlenecks: activeBottlenecks, healthScore: this.calculateHealthScore(currentMetrics), recommendations: this.generateSystemRecommendations( currentMetrics, recentTrends ), historicalData: { totalSnapshots: this.performanceHistory.length, timespan: this.calculateHistoryTimespan(), averagePerformance: this.getRecentPerformanceAverage(), }, }; } private generateSystemRecommendations( metrics: PerformanceMetrics, trends: any ): string[] { const recommendations: string[] = []; // Memory recommendations if (metrics.memoryUsage > 80) { recommendations.push( 'High memory usage detected - consider enabling compression' ); recommendations.push('Review memory retention policies'); } // Performance recommendations if (metrics.responseTime > this.config.alertThresholds.responseTime) { recommendations.push( 'Response times are elevated - investigate bottlenecks' ); recommendations.push('Consider scaling up agent pool'); } // Throughput recommendations if (metrics.throughput < 5) { recommendations.push( 'Low throughput detected - check for blocking operations' ); } // Trend-based recommendations if (trends.memoryTrend === 'increasing') { recommendations.push( 'Memory usage trend is increasing - monitor for leaks' ); } if (trends.errorTrend === 'increasing') { recommendations.push( 'Error rate is trending upward - investigate root causes' ); } return recommendations; } private calculateHistoryTimespan(): number { if (this.performanceHistory.length < 2) return 0; const oldest = this.performanceHistory[0].timestamp.getTime(); const newest = this.performanceHistory[ this.performanceHistory.length - 1 ].timestamp.getTime(); return newest - oldest; } async getMetrics(): Promise { const currentMetrics = await this.metricsCollector.collectMetrics(); return { isMonitoring: this.isMonitoring, currentMetrics, historySize: this.performanceHistory.length, metricsCollectionEnabled: this.config.metricsCollection, bottleneckDetectionEnabled: this.config.bottleneckDetection, autoOptimizationEnabled: this.config.autoOptimization, alertThresholds: this.config.alertThresholds, }; } async shutdown(): Promise { try { // Stop monitoring if (this.monitoringTimer) { clearInterval(this.monitoringTimer); this.monitoringTimer = null; } this.isMonitoring = false; // Shutdown subsystems await this.metricsCollector.shutdown(); await this.bottleneckDetector.shutdown(); await this.trendAnalyzer.shutdown(); await this.alertManager.shutdown(); // Clear history this.performanceHistory.length = 0; return { success: true, message: 'Performance Analyzer shutdown completed', }; } catch (error) { return { success: false, message: `Shutdown failed: ${error.message}`, error: error, }; } } } // Supporting Classes and Interfaces interface PerformanceSnapshot { timestamp: Date; metrics: PerformanceMetrics; systemLoad: number; healthScore: number; } interface TaskAnalysis { taskId: string; complexity: number; estimatedResources: ResourceRequirements; riskFactors: RiskFactor[]; recommendations: string[]; confidence: number; } interface ResourceRequirements { cpuEstimate: number; memoryEstimate: number; timeEstimate: number; agentsRequired: number; } interface RiskFactor { type: string; severity: 'low' | 'medium' | 'high' | 'critical'; description: string; mitigation: string; } interface OptimizationStrategy { target: string; severity: string; actions: string[]; expectedImpact: 'low' | 'medium' | 'high'; } interface PerformanceReport { timestamp: Date; currentMetrics: PerformanceMetrics; trends: any; bottlenecks: Bottleneck[]; healthScore: number; recommendations: string[]; historicalData: { totalSnapshots: number; timespan: number; averagePerformance: PerformanceMetrics; }; } class MetricsCollector { // Rolling window of latency samples (ms) for percentile calculations private latencySamples: number[] = []; private readonly maxSamples = 1000; // Request/error counters and the timestamp of the last throughput window private requestCount = 0; private errorCount = 0; private activeConnections = 0; private queueLength = 0; private lastThroughputWindowStart: bigint = process.hrtime.bigint(); private requestsInWindow = 0; // CPU measurement state: snapshot of (idle, total) times per core private prevCpuTimes: Array<{ idle: number; total: number }> = []; async initialize(): Promise { this.prevCpuTimes = this.snapshotCpuTimes(); } /** * Record the completion of a single request so that throughput, latency, * and error-rate counters stay accurate between collectMetrics() calls. * * @param latencyMs Wall-clock duration of the request in milliseconds. * @param isError Whether the request ended with an error. */ recordRequest(latencyMs: number, isError = false): void { this.requestCount++; this.requestsInWindow++; if (isError) this.errorCount++; this.latencySamples.push(latencyMs); if (this.latencySamples.length > this.maxSamples) { this.latencySamples.shift(); } } /** Adjust the active-connection gauge (positive to add, negative to remove). */ adjustConnections(delta: number): void { this.activeConnections = Math.max(0, this.activeConnections + delta); } /** Set the current task-queue depth. */ setQueueLength(length: number): void { this.queueLength = Math.max(0, length); } async collectMetrics(): Promise { const memUsage = process.memoryUsage(); const cpuUsage = this.getCPUUsage(); // --- throughput (requests per second over the elapsed window) --- const nowNs = process.hrtime.bigint(); const elapsedSec = Number(nowNs - this.lastThroughputWindowStart) / 1_000_000_000; const throughput = elapsedSec > 0 ? this.requestsInWindow / elapsedSec : 0; // Reset the throughput window this.requestsInWindow = 0; this.lastThroughputWindowStart = nowNs; // --- error rate --- const errorRate = this.requestCount > 0 ? this.errorCount / this.requestCount : 0; // --- p50 response time (median) as the primary responseTime metric --- const responseTime = this.percentile(this.latencySamples, 50); return { responseTime, throughput, errorRate, memoryUsage: (memUsage.heapUsed / memUsage.heapTotal) * 100, cpuUsage, activeConnections: this.activeConnections, queueLength: this.queueLength, }; } // ----------------------------------------------------------------------- // Private helpers // ----------------------------------------------------------------------- private snapshotCpuTimes(): Array<{ idle: number; total: number }> { return os.cpus().map(cpu => { const times = cpu.times; const total = times.user + times.nice + times.sys + times.idle + times.irq; return { idle: times.idle, total }; }); } private getCPUUsage(): number { const current = this.snapshotCpuTimes(); if (this.prevCpuTimes.length !== current.length) { // First call or CPU count changed – baseline only, return 0. this.prevCpuTimes = current; return 0; } let idleDelta = 0; let totalDelta = 0; for (let i = 0; i < current.length; i++) { idleDelta += current[i].idle - this.prevCpuTimes[i].idle; totalDelta += current[i].total - this.prevCpuTimes[i].total; } this.prevCpuTimes = current; if (totalDelta === 0) return 0; return ((totalDelta - idleDelta) / totalDelta) * 100; } /** * Compute the p-th percentile of an array of numbers. * Returns 0 when the array is empty. */ private percentile(samples: number[], p: number): number { if (samples.length === 0) return 0; const sorted = [...samples].sort((a, b) => a - b); const index = Math.ceil((p / 100) * sorted.length) - 1; return sorted[Math.max(0, index)]; } async shutdown(): Promise { this.latencySamples = []; this.requestCount = 0; this.errorCount = 0; this.activeConnections = 0; this.queueLength = 0; this.requestsInWindow = 0; this.prevCpuTimes = []; } } class BottleneckDetector { private config: PerformanceConfig; private currentBottlenecks: Bottleneck[] = []; constructor(config: PerformanceConfig) { this.config = config; } async initialize(): Promise { // Initialize bottleneck detection algorithms } async detectBottlenecks( snapshot: PerformanceSnapshot ): Promise { const bottlenecks: Bottleneck[] = []; const metrics = snapshot.metrics; // Memory bottleneck if (metrics.memoryUsage > this.config.alertThresholds.memoryUsage) { bottlenecks.push({ type: 'memory', severity: this.calculateSeverity( 'memory', metrics.memoryUsage, this.config.alertThresholds.memoryUsage ), description: `High memory usage: ${metrics.memoryUsage.toFixed(1)}%`, affectedComponents: ['agents', 'memory-manager', 'neural-pipeline'], suggestedActions: [ 'Enable compression', 'Clear unused memories', 'Optimize cache', ], detectedAt: snapshot.timestamp, }); } // Response time bottleneck if (metrics.responseTime > this.config.alertThresholds.responseTime) { bottlenecks.push({ type: 'network', severity: this.calculateSeverity( 'response-time', metrics.responseTime, this.config.alertThresholds.responseTime ), description: `High response time: ${metrics.responseTime}ms`, affectedComponents: ['agents', 'coordination', 'communication'], suggestedActions: [ 'Scale agents', 'Optimize coordination', 'Enable batching', ], detectedAt: snapshot.timestamp, }); } // Error rate bottleneck if (metrics.errorRate > this.config.alertThresholds.errorRate) { bottlenecks.push({ type: 'agent', severity: this.calculateSeverity( 'error-rate', metrics.errorRate, this.config.alertThresholds.errorRate ), description: `High error rate: ${(metrics.errorRate * 100).toFixed(2)}%`, affectedComponents: ['agents', 'task-execution', 'coordination'], suggestedActions: [ 'Check agent health', 'Investigate failures', 'Implement retry logic', ], detectedAt: snapshot.timestamp, }); } // Queue length bottleneck if (metrics.queueLength > 25) { bottlenecks.push({ type: 'task-queue', severity: this.calculateSeverity( 'queue-length', metrics.queueLength, 25 ), description: `High queue length: ${metrics.queueLength}`, affectedComponents: ['task-orchestrator', 'agents', 'load-balancer'], suggestedActions: [ 'Increase parallelism', 'Scale agents', 'Optimize scheduling', ], detectedAt: snapshot.timestamp, }); } // Update current bottlenecks this.currentBottlenecks = bottlenecks; return bottlenecks; } private calculateSeverity( type: string, current: number, threshold: number ): 'low' | 'medium' | 'high' | 'critical' { const ratio = current / threshold; if (ratio >= 2.0) return 'critical'; if (ratio >= 1.5) return 'high'; if (ratio >= 1.2) return 'medium'; return 'low'; } async getCurrentBottlenecks(): Promise { return [...this.currentBottlenecks]; } async shutdown(): Promise { this.currentBottlenecks = []; } } class TrendAnalyzer { async initialize(): Promise { // Initialize trend analysis } async analyzeTrends(history: PerformanceSnapshot[]): Promise { if (history.length < 5) { return { insufficient_data: true }; } const recent = history.slice(-20); // Analyze last 20 snapshots return { memoryTrend: this.calculateTrend(recent.map(s => s.metrics.memoryUsage)), responseTrend: this.calculateTrend( recent.map(s => s.metrics.responseTime) ), errorTrend: this.calculateTrend(recent.map(s => s.metrics.errorRate)), throughputTrend: this.calculateTrend( recent.map(s => s.metrics.throughput) ), healthTrend: this.calculateTrend(recent.map(s => s.healthScore)), timespan: recent[recent.length - 1].timestamp.getTime() - recent[0].timestamp.getTime(), }; } private calculateTrend( values: number[] ): 'increasing' | 'decreasing' | 'stable' { if (values.length < 3) return 'stable'; const first = values.slice(0, Math.floor(values.length / 2)); const last = values.slice(Math.floor(values.length / 2)); const firstAvg = first.reduce((sum, val) => sum + val, 0) / first.length; const lastAvg = last.reduce((sum, val) => sum + val, 0) / last.length; const change = (lastAvg - firstAvg) / firstAvg; if (change > 0.1) return 'increasing'; if (change < -0.1) return 'decreasing'; return 'stable'; } async shutdown(): Promise { // Cleanup trend analysis } } class AlertManager { private config: PerformanceConfig; constructor(config: PerformanceConfig) { this.config = config; } async initialize(): Promise { // Initialize alert management } async checkAlerts(snapshot: PerformanceSnapshot, trends: any): Promise { const alerts: any[] = []; // Check threshold alerts if ( snapshot.metrics.responseTime > this.config.alertThresholds.responseTime ) { alerts.push({ type: 'threshold', metric: 'responseTime', current: snapshot.metrics.responseTime, threshold: this.config.alertThresholds.responseTime, severity: 'high', }); } // Check trend alerts if (trends.errorTrend === 'increasing') { alerts.push({ type: 'trend', metric: 'errorRate', trend: 'increasing', severity: 'medium', }); } // Emit alerts for (const alert of alerts) { this.emitAlert(alert); } } private emitAlert(alert: any): void { console.warn( `ALERT [${alert.severity.toUpperCase()}]: ${alert.type} - ${alert.metric}`, alert ); } async shutdown(): Promise { // Cleanup alert management } }