import { MetricServiceClient } from '@google-cloud/monitoring'; import type { protos } from '@google-cloud/monitoring'; /** * Google Cloud Monitoring integration for AI Universe * Exports singleton instance for shared monitoring across all services * @packageDocumentation */ /** * Metric types supported by the monitoring service */ export type MetricType = | 'tool_call_latency' | 'tool_call_count' | 'tool_call_errors' | 'http_request_latency' | 'http_request_count' | 'http_request_errors' | 'http_request_size_bytes' | 'http_response_size_bytes' | 'active_requests' | 'token_usage' | 'streaming_chunks' | 'auth_attempts' | 'rate_limit_hits'; /** * Labels for metrics to enable filtering and grouping */ export interface MetricLabels { tool_name?: string; model_name?: string; operation?: string; domain?: string; status_code?: string; status?: string; error_type?: string; user_id?: string; endpoint?: string; [key: string]: string | undefined; } /** * Logger interface for monitoring service */ export interface MonitoringLogger { info: (message: string, meta?: any) => void; warn: (message: string, meta?: any) => void; error: (message: string, meta?: any) => void; debug: (message: string, meta?: any) => void; } /** * Configuration for monitoring service */ export interface MonitoringConfig { projectId: string; enabled: boolean; writeInterval?: number; // Batch write interval in ms maxBatchSize?: number; logger?: MonitoringLogger; metricPrefix?: string; maxRetries?: number; // Maximum initialization retry attempts retryDelay?: number; // Initial retry delay in ms (exponential backoff) maxClockSkewMs?: number; // Maximum allowed clock skew in ms (default: 15 minutes) minSamplingPeriodMs?: number; // Minimum sampling period for metrics in ms (default: 1000ms = 1 second) } /** * Pending metric to be written */ interface PendingMetric { type: MetricType; value: number; labels: MetricLabels; timestamp: Date; // For distribution metrics (latency) isDistribution?: boolean; // Original timestamp before bucketing (used for gauge aggregation) originalTimestamp?: Date; } /** * Default console logger with async (non-blocking) writes * Uses setImmediate to ensure logging never blocks the event loop */ const defaultLogger: MonitoringLogger = { info: (message: string, meta?: any) => { setImmediate(() => console.info(message, meta)); }, warn: (message: string, meta?: any) => { setImmediate(() => console.warn(message, meta)); }, error: (message: string, meta?: any) => { setImmediate(() => console.error(message, meta)); }, debug: (message: string, meta?: any) => { setImmediate(() => console.debug(message, meta)); } }; /** * Google Cloud Monitoring service * * Provides centralized monitoring for: * - Tool call latency and success/failure rates * - HTTP request latency and error rates * - Token usage tracking * - Rate limiting and authentication metrics * - Active request gauges */ export class MonitoringService { private client: MetricServiceClient | null = null; private config: MonitoringConfig; private initialized: boolean = false; private initializationAttempts: number = 0; private pendingMetrics: PendingMetric[] = []; private writeIntervalHandle: NodeJS.Timeout | null = null; private readonly logger: MonitoringLogger; private readonly metricPrefix: string; constructor(config?: Partial) { // Detect if we're in a GCP environment with ADC const hasADC = this.detectApplicationDefaultCredentials(); this.config = { projectId: config?.projectId || process.env.GCP_PROJECT_ID || process.env.GOOGLE_CLOUD_PROJECT || 'ai-universe-2025', enabled: config?.enabled ?? hasADC, writeInterval: config?.writeInterval || 10000, // 10 seconds maxBatchSize: config?.maxBatchSize || 100, maxRetries: config?.maxRetries ?? 3, retryDelay: config?.retryDelay || 1000, // 1 second initial delay maxClockSkewMs: config?.maxClockSkewMs ?? 900000, // 15 minutes default minSamplingPeriodMs: config?.minSamplingPeriodMs ?? 1000 // 1 second default (GCP minimum sampling period) }; this.logger = config?.logger || defaultLogger; this.metricPrefix = config?.metricPrefix || 'custom.googleapis.com/ai_universe'; } /** * Detect if Application Default Credentials are available * Supports: * - GOOGLE_APPLICATION_CREDENTIALS (local dev with service account) * - Cloud Run Workload Identity (GOOGLE_CLOUD_PROJECT or K_SERVICE) * - GCP Compute Engine / GKE metadata server * * Disables monitoring in CI environments where GCP env vars are set * for configuration but no actual credentials are available. */ private detectApplicationDefaultCredentials(): boolean { // Disable in CI environments (GitHub Actions, GitLab CI, etc.) // These set GCP env vars for testing but don't have real credentials if (process.env.CI || process.env.GITHUB_ACTIONS || process.env.GITLAB_CI) { return false; } // Explicit service account key file if (process.env.GOOGLE_APPLICATION_CREDENTIALS) { return true; } // Cloud Run or GCP environment if (process.env.GOOGLE_CLOUD_PROJECT || process.env.GCP_PROJECT) { return true; } // Cloud Run specific environment variable if (process.env.K_SERVICE) { return true; } // GKE or Compute Engine (KUBERNETES_SERVICE_HOST indicates GKE) if (process.env.KUBERNETES_SERVICE_HOST) { return true; } return false; } /** * Initialize the monitoring service with retry logic */ private async ensureInitialized(): Promise { if (this.initialized) return; // If explicitly disabled in config, mark as initialized and skip if (this.config.enabled === false) { this.logger.info('Monitoring service explicitly disabled in configuration'); this.initialized = true; return; } try { this.initializationAttempts++; // Initialize Google Cloud Monitoring client (uses ADC) this.client = new MetricServiceClient(); // Validate credentials by testing projectPath generation // This will throw if credentials are invalid const projectPath = this.client.projectPath(this.config.projectId); if (!projectPath) { throw new Error('Invalid project path - credentials may be unavailable'); } // Start batch writing interval this.startBatchWriting(); this.logger.info('Monitoring service initialized successfully', { projectId: this.config.projectId, writeInterval: this.config.writeInterval, attempts: this.initializationAttempts, adcDetected: this.detectApplicationDefaultCredentials() }); this.initialized = true; this.initializationAttempts = 0; // Reset on success } catch (error) { const maxRetries = this.config.maxRetries || 3; // Enhanced error details for initialization failures const errorDetails: Record = { errorMessage: error instanceof Error ? error.message : String(error), errorStack: error instanceof Error ? error.stack : undefined, errorCode: (error as any).code, errorName: error instanceof Error ? error.constructor.name : typeof error, projectId: this.config.projectId, attempt: this.initializationAttempts, maxRetries, adcDetected: this.detectApplicationDefaultCredentials() }; if (this.initializationAttempts < maxRetries) { // Transient failure - log warning and allow retry const retryDelay = (this.config.retryDelay || 1000) * Math.pow(2, this.initializationAttempts - 1); this.logger.warn('Failed to initialize monitoring service, will retry', { ...errorDetails, retryInMs: retryDelay }); // Wait before the next retry attempt to avoid hammering the API await new Promise(resolve => setTimeout(resolve, retryDelay)); // Don't mark as initialized - allow retry on next metric call } else { // Permanent failure after max retries this.logger.error('Failed to initialize monitoring service after max retries, monitoring permanently disabled', errorDetails); this.config = { ...this.config, enabled: false }; this.initialized = true; // Prevent further retries } } } /** * Start batch writing of metrics */ private startBatchWriting(): void { if (!this.config.enabled || this.writeIntervalHandle) return; this.writeIntervalHandle = setInterval(() => { void this.flushMetrics(); }, this.config.writeInterval); // Note: Shutdown should be called explicitly by the application // Do not register signal handlers here to avoid conflicts with application-level shutdown logic } /** * Record a metric value */ async recordMetric(type: MetricType, value: number, labels: MetricLabels = {}): Promise { await this.ensureInitialized(); if (!this.config.enabled) return; try { // Add to pending metrics batch this.pendingMetrics.push({ type, value, labels, timestamp: new Date() }); // Flush if batch is full if (this.pendingMetrics.length >= this.config.maxBatchSize!) { await this.flushMetrics(); } } catch (error) { // Never throw - monitoring failures should not break the app this.logger.debug('Failed to record metric', { type, error: error instanceof Error ? error.message : 'Unknown error' }); } } /** * Record latency for an operation */ async recordLatency(type: MetricType, durationMs: number, labels: MetricLabels = {}): Promise { await this.recordMetric(type, durationMs, labels); } /** * Increment a counter metric */ async incrementCounter(type: MetricType, labels: MetricLabels = {}): Promise { await this.recordMetric(type, 1, labels); } /** * Record a gauge value (e.g., active requests) */ async recordGauge(type: MetricType, value: number, labels: MetricLabels = {}): Promise { await this.recordMetric(type, value, labels); } /** * Round timestamp to the nearest sampling period bucket * This prevents Google Cloud Monitoring INVALID_ARGUMENT errors for duplicate timestamps */ private roundTimestampToBucket(timestamp: Date): Date { const samplingPeriodMs = this.config.minSamplingPeriodMs || 1000; const timestampMs = timestamp.getTime(); const bucketMs = Math.floor(timestampMs / samplingPeriodMs) * samplingPeriodMs; return new Date(bucketMs); } /** * Flush pending metrics to Google Cloud Monitoring */ async flushMetrics(): Promise { if (!this.config.enabled || !this.client || this.pendingMetrics.length === 0) { return; } const metricsToWrite = [...this.pendingMetrics]; this.pendingMetrics = []; try { const projectPath = this.client.projectPath(this.config.projectId); // Group metrics by type for efficient batching const metricsByType = new Map(); for (const metric of metricsToWrite) { if (!metricsByType.has(metric.type)) { metricsByType.set(metric.type, []); } metricsByType.get(metric.type)!.push(metric); } // Write each metric type as a batch let aggregatedCount = 0; for (const [type, metrics] of metricsByType) { // Aggregate metrics with same labels AND same time bucket // Google Cloud rejects multiple TimeSeries with timestamps within the minimum sampling period const aggregatedMetrics = new Map(); for (const metric of metrics) { // Round timestamp to the nearest sampling period bucket const bucketedTimestamp = this.roundTimestampToBucket(metric.timestamp); // Create a unique key from labels + bucketed timestamp const labelKey = JSON.stringify( Object.entries(metric.labels) .sort(([a], [b]) => a.localeCompare(b)) // Sort for consistent key .map(([k, v]) => `${k}:${v}`) ); const aggregationKey = `${labelKey}|${bucketedTimestamp.getTime()}`; const existing = aggregatedMetrics.get(aggregationKey); if (!existing) { // First metric in this bucket - use it with bucketed timestamp aggregatedMetrics.set(aggregationKey, { ...metric, timestamp: bucketedTimestamp, originalTimestamp: metric.timestamp // Store original for gauge comparison }); } else { // Aggregate with existing metric in the same bucket const metricKind = this.getMetricKind(type); const isDistribution = metric.isDistribution || this.isLatencyMetric(type); if (isDistribution) { // For distributions (latency, size histograms), keep the first one // Alternative: could merge distribution buckets, but that's complex // The first sample is representative of the bucket period continue; } else if (metricKind === 'CUMULATIVE') { // For counters, sum the values (e.g., tool_call_count) existing.value += metric.value; } else if (metricKind === 'GAUGE') { // For gauges (e.g., active_requests), use the latest value // Compare original timestamps to determine which is latest const existingOriginal = existing.originalTimestamp || existing.timestamp; if (metric.timestamp > existingOriginal) { existing.value = metric.value; existing.originalTimestamp = metric.timestamp; } } } } const timeSeries = Array.from(aggregatedMetrics.values()) .map(metric => this.createTimeSeries(type, metric)); aggregatedCount += timeSeries.length; await this.client.createTimeSeries({ name: projectPath, timeSeries }); } this.logger.debug('Flushed metrics to Google Cloud Monitoring', { count: metricsToWrite.length, aggregatedCount, // Number of time series after aggregation types: Array.from(metricsByType.keys()) }); } catch (error) { // Check if this is the duplicate timestamp error const errorMessage = error instanceof Error ? error.message : String(error); const errorCode = (error as any).code; const isInvalidArgument = errorCode === 3 || errorMessage.includes('INVALID_ARGUMENT'); const isDuplicateTimestamp = errorMessage.includes('written more frequently than the maximum sampling period'); const isFutureTimestamp = errorMessage.includes('into the future') || errorMessage.includes('after the present time'); if (isInvalidArgument && isDuplicateTimestamp) { // Graceful handling for duplicate timestamp errors - log summary only this.logger.warn('Monitoring flush skipped duplicate timestamps', { errorCode, metricsCount: metricsToWrite.length, metricTypes: Array.from(new Set(metricsToWrite.map(m => m.type))), projectId: this.config.projectId, hint: 'Consider increasing minSamplingPeriodMs or adjusting metric recording frequency' }); } else if (isInvalidArgument && isFutureTimestamp) { // Graceful handling for clock skew errors - metrics work but some may be rejected this.logger.warn('Monitoring flush skipped future timestamps (clock skew detected)', { metricsCount: metricsToWrite.length, metricTypes: Array.from(new Set(metricsToWrite.map(m => m.type))), hint: 'System clock is ahead of GCP servers - some metrics may be delayed' }); } else { // Enhanced error logging for other failures const errorDetails: Record = { errorMessage, errorCode, errorName: error instanceof Error ? error.constructor.name : typeof error, projectId: this.config.projectId, metricsCount: metricsToWrite.length, metricTypes: Array.from(new Set(metricsToWrite.map(m => m.type))) }; // Capture additional GCP-specific error properties (but not full stack trace) if (error && typeof error === 'object') { const grcpError = error as any; if (grcpError.details) errorDetails.errorDetails = grcpError.details; if (grcpError.statusCode) errorDetails.statusCode = grcpError.statusCode; } this.logger.error('Failed to flush metrics to Google Cloud Monitoring', errorDetails); } // Don't re-add to pending - accept the loss to prevent memory buildup } } /** * Create exponential buckets for distribution metrics (latency) * Buckets: 0-10ms, 10-100ms, 100ms-1s, 1s-10s, 10s-100s */ private createExponentialBuckets(): protos.google.api.IDistribution['bucketOptions'] { return { exponentialBuckets: { numFiniteBuckets: 50, // Number of buckets growthFactor: 1.5, // Each bucket is 1.5x the previous scale: 1 // Starting value in milliseconds } }; } /** * Create a distribution value for latency metrics */ private createDistributionValue(latencyMs: number): protos.google.api.IDistribution { const bucketOptions = this.createExponentialBuckets(); if (!bucketOptions || !bucketOptions.exponentialBuckets) { throw new Error('Exponential buckets configuration is missing'); } const expBuckets = bucketOptions.exponentialBuckets; const numBuckets = (expBuckets.numFiniteBuckets || 0) + 2; // +2 for underflow/overflow // Calculate which bucket this value falls into const bucketCounts = new Array(numBuckets).fill(0); const scale = expBuckets.scale || 1; const growthFactor = expBuckets.growthFactor || 1.5; // Find the correct bucket let bucketIndex = 0; if (latencyMs < scale) { bucketIndex = 0; // Underflow bucket } else { let upperBound = scale; for (let i = 0; i < (expBuckets.numFiniteBuckets || 0); i++) { upperBound *= growthFactor; if (latencyMs < upperBound) { bucketIndex = i + 1; break; } } if (bucketIndex === 0) { bucketIndex = numBuckets - 1; // Overflow bucket } } bucketCounts[bucketIndex] = 1; return { count: 1, mean: latencyMs, sumOfSquaredDeviation: 0, // For single values bucketOptions, bucketCounts }; } /** * Adjust timestamp to prevent clock skew issues with GCP Monitoring * GCP rejects timestamps >5min in the future, so we cap all future timestamps to "now" * * @param timestamp Original timestamp * @returns Adjusted timestamp (capped to current time if in future) */ private adjustTimestampForClockSkew(timestamp: Date): Date { const now = new Date(); const skewMs = timestamp.getTime() - now.getTime(); // If timestamp is in the future, cap it to "now" (GCP allows max 5min future) if (skewMs > 0) { // Cap to "now" to ensure we're always within GCP's limits const adjustedTimestamp = now; // Only log if skew is significant (>10 seconds) if (skewMs > 10000) { setImmediate(() => { this.logger.debug('Clock skew detected: future timestamp adjusted to now', { skewSeconds: Math.round(skewMs / 1000), hint: 'System clock may be ahead - metrics timestamps normalized to GCP server time' }); }); } return adjustedTimestamp; } return timestamp; } /** * Create a time series object for Google Cloud Monitoring */ private createTimeSeries(type: MetricType, metric: PendingMetric): protos.google.monitoring.v3.ITimeSeries { const metricKind = this.getMetricKind(type); const isDistribution = metric.isDistribution || this.isLatencyMetric(type); const valueType = isDistribution ? 'DISTRIBUTION' : 'DOUBLE'; // Convert labels to Google Cloud Monitoring format const labels: { [key: string]: string } = {}; for (const [key, value] of Object.entries(metric.labels)) { if (value !== undefined) { labels[key] = String(value); } } // Adjust timestamp to prevent clock skew issues const adjustedTimestamp = this.adjustTimestampForClockSkew(metric.timestamp); const endTimeSeconds = Math.floor(adjustedTimestamp.getTime() / 1000); const endTimeNanos = (adjustedTimestamp.getTime() % 1000) * 1000000; // For CUMULATIVE metrics, we need both startTime and endTime // For GAUGE, only endTime is required const interval: protos.google.monitoring.v3.ITimeInterval = { endTime: { seconds: endTimeSeconds, nanos: endTimeNanos } }; // Add startTime for CUMULATIVE metrics (marks when counter started) if (metricKind === 'CUMULATIVE') { interval.startTime = { seconds: endTimeSeconds - 60, // Counter started 60 seconds ago (arbitrary, required by API) nanos: endTimeNanos }; } // Create point value const value: protos.google.monitoring.v3.ITypedValue = isDistribution ? { distributionValue: this.createDistributionValue(metric.value) } : { doubleValue: metric.value }; return { metric: { type: `${this.metricPrefix}/${type}`, labels }, resource: { type: 'global', labels: { project_id: this.config.projectId } }, metricKind, valueType, points: [ { interval, value } ] }; } /** * Check if a metric type should use DISTRIBUTION (latency, size histograms) */ private isLatencyMetric(type: MetricType): boolean { const distributionMetrics: MetricType[] = [ 'tool_call_latency', 'http_request_latency', 'http_request_size_bytes', 'http_response_size_bytes' ]; return distributionMetrics.includes(type); } /** * Get the metric kind for a metric type */ private getMetricKind(type: MetricType): 'GAUGE' | 'CUMULATIVE' { // DISTRIBUTION metrics (latency, size) MUST be GAUGE (cannot change once created) const gaugeMetrics: MetricType[] = [ 'active_requests', 'tool_call_latency', // DISTRIBUTION - must be GAUGE 'http_request_latency', // DISTRIBUTION - must be GAUGE 'http_request_size_bytes', // DISTRIBUTION - must be GAUGE 'http_response_size_bytes' // DISTRIBUTION - must be GAUGE ]; // All counters MUST be CUMULATIVE (Google Cloud doesn't support DELTA for custom metrics) const cumulativeMetrics: MetricType[] = [ 'token_usage', 'tool_call_count', 'http_request_count', 'tool_call_errors', 'http_request_errors', 'streaming_chunks', 'auth_attempts', 'rate_limit_hits' ]; if (gaugeMetrics.includes(type)) { return 'GAUGE'; } else if (cumulativeMetrics.includes(type)) { return 'CUMULATIVE'; } else { // Default to GAUGE for unknown metrics return 'GAUGE'; } } /** * Helper: Measure and record execution time of an async function */ async measureAsync( metricType: MetricType, labels: MetricLabels, fn: () => Promise ): Promise { const startTime = Date.now(); let success = true; try { const result = await fn(); return result; } catch (error) { success = false; throw error; } finally { const duration = Date.now() - startTime; await this.recordLatency(metricType, duration, { ...labels, status: success ? 'success' : 'error' }); } } /** * Helper: Measure and record execution time of a sync function */ measure( metricType: MetricType, labels: MetricLabels, fn: () => T ): T { const startTime = Date.now(); let success = true; try { const result = fn(); return result; } catch (error) { success = false; throw error; } finally { const duration = Date.now() - startTime; void this.recordLatency(metricType, duration, { ...labels, status: success ? 'success' : 'error' }); } } /** * Shutdown the monitoring service gracefully */ async shutdown(): Promise { this.logger.info('Shutting down monitoring service...'); // Stop batch writing if (this.writeIntervalHandle) { clearInterval(this.writeIntervalHandle); this.writeIntervalHandle = null; } // Flush remaining metrics await this.flushMetrics(); // Close client if (this.client) { await this.client.close(); this.client = null; } this.logger.info('Monitoring service shut down successfully'); } /** * Get monitoring statistics (for debugging/health checks) */ getStats(): { enabled: boolean; pendingMetrics: number; projectId: string } { return { enabled: this.config.enabled, pendingMetrics: this.pendingMetrics.length, projectId: this.config.projectId }; } } // Export singleton instance for convenience (must be before helper functions that use it) export const monitoringService = new MonitoringService(); /** * Helper function for measuring tool calls * Uses the singleton monitoringService instance to avoid memory leaks */ export interface ToolMonitoringOptions { labels?: MetricLabels; logger?: MonitoringLogger; } export async function withToolMonitoring( toolName: string, operation: string, fn: () => Promise, options?: ToolMonitoringOptions ): Promise { const startTime = Date.now(); let success = true; let errorType: string | undefined; const logger = options?.logger; const baseLabels: MetricLabels = { tool_name: toolName, operation, ...(options?.labels ?? {}) }; try { const result = await fn(); void monitoringService.incrementCounter('tool_call_count', { ...baseLabels, status: 'success' }).catch(err => logger?.debug('Monitoring error (non-critical):', err)); return result; } catch (error) { success = false; errorType = error instanceof Error ? error.constructor.name : 'UnknownError'; // Record error to main counter with status='error' (unified counter pattern) void monitoringService.incrementCounter('tool_call_count', { ...baseLabels, status: 'error', error_type: errorType }).catch(err => logger?.debug('Monitoring error (non-critical):', err)); // Also record to separate error metric for backward compatibility void monitoringService.incrementCounter('tool_call_errors', { ...baseLabels, error_type: errorType }).catch(err => logger?.debug('Monitoring error (non-critical):', err)); throw error; } finally { const duration = Date.now() - startTime; void monitoringService.recordLatency('tool_call_latency', duration, { ...baseLabels, status: success ? 'success' : 'error', ...(errorType && { error_type: errorType }) }).catch(err => logger?.debug('Monitoring error (non-critical):', err)); } } /** * Helper function for measuring HTTP calls * Uses the singleton monitoringService instance to avoid memory leaks */ export async function withHttpMonitoring( domain: string, operation: string, fn: () => Promise, getStatusCode?: () => string | undefined, logger?: MonitoringLogger ): Promise { const startTime = Date.now(); let success = true; let statusCode: string | undefined; try { const result = await fn(); statusCode = getStatusCode?.(); void monitoringService.incrementCounter('http_request_count', { domain, operation, status: 'success', ...(statusCode && { status_code: statusCode }) }).catch(err => logger?.debug('Monitoring error (non-critical):', err)); return result; } catch (error) { success = false; statusCode = getStatusCode?.() || '0'; // Record error to main counter with status='error' (unified counter pattern) void monitoringService.incrementCounter('http_request_count', { domain, operation, status: 'error', status_code: statusCode }).catch(err => logger?.debug('Monitoring error (non-critical):', err)); // Also record to separate error metric for backward compatibility void monitoringService.incrementCounter('http_request_errors', { domain, operation, status_code: statusCode }).catch(err => logger?.debug('Monitoring error (non-critical):', err)); throw error; } finally { const duration = Date.now() - startTime; void monitoringService.recordLatency('http_request_latency', duration, { domain, operation, status: success ? 'success' : 'error', ...(statusCode && { status_code: statusCode }) }).catch(err => logger?.debug('Monitoring error (non-critical):', err)); } }