import { Config } from '../types/config'; // Sensitive data patterns for masking const SENSITIVE_PATTERNS = { // Password fields password: /password/i, passwd: /passwd/i, pwd: /pwd/i, // Email addresses email: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, // Credit card numbers (various formats) creditCard: /\b(?:\d{4}[-\s]?){3}\d{4}\b/g, // Phone numbers (US and international formats) phone: /\b(?:\+?1[-.\s]?)?\(?([0-9]{3})\)?[-.\s]?([0-9]{3})[-.\s]?([0-9]{4})\b/g, // Social Security Numbers ssn: /\b\d{3}-?\d{2}-?\d{4}\b/g, // API keys and tokens (common patterns) apiKey: /\b[A-Za-z0-9]{32,}\b/g, jwt: /\beyJ[A-Za-z0-9-_]+\.[A-Za-z0-9-_]+\.[A-Za-z0-9-_]*\b/g, // IP addresses (when configured as sensitive) ipAddress: /\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b/g, // Database connection strings connectionString: /(mongodb|postgresql|mysql|redis):\/\/[^\/\s]+/gi }; // Field names that should be masked (case insensitive) const SENSITIVE_FIELD_NAMES = [ 'password', 'passwd', 'pwd', 'secret', 'token', 'key', 'auth', 'credit_card', 'creditcard', 'ccnum', 'card_number', 'ssn', 'social_security', 'social_security_number', 'phone', 'phone_number', 'mobile', 'telephone', 'email', 'email_address', 'mail', 'api_key', 'apikey', 'access_key', 'secret_key', 'jwt', 'bearer', 'authorization', 'pin', 'cvv', 'cvc', 'security_code' ]; export interface MaskingConfig { enabled?: boolean; // Enable data masking (default: true) maskingChar?: string; // Character to use for masking (default: '*') preserveLength?: boolean; // Preserve original field length (default: false) showLastChars?: number; // Show last N characters (default: 0) customPatterns?: { [key: string]: RegExp }; // Custom patterns to mask customFields?: string[]; // Custom field names to mask exemptFields?: string[]; // Fields to exclude from masking maskEmails?: boolean; // Mask email addresses (default: true) maskIPs?: boolean; // Mask IP addresses (default: false) maskConnectionStrings?: boolean; // Mask database connection strings (default: true) } /** * Mask sensitive data in API logs and other data * Enhanced with memory-safe processing for large datasets */ export function maskSensitiveData(data: any, config?: MaskingConfig): any { const maskingConfig: Required = { enabled: true, maskingChar: '*', preserveLength: false, showLastChars: 0, customPatterns: {}, customFields: [], exemptFields: [], maskEmails: true, maskIPs: false, maskConnectionStrings: true, ...config }; if (!maskingConfig.enabled) { return data; } // Memory monitoring for large data processing const memBefore = process.memoryUsage().heapUsed / 1024 / 1024; if (memBefore > 100) { console.log(`⚠️ High memory usage before masking: ${memBefore.toFixed(2)}MB`); } let result: any; if (typeof data === 'string') { result = maskString(data, maskingConfig); } else if (Array.isArray(data)) { result = maskArraySafe(data, maskingConfig); } else if (typeof data === 'object' && data !== null) { result = maskObjectSafe(data, maskingConfig); } else { result = data; } // Memory check after masking const memAfter = process.memoryUsage().heapUsed / 1024 / 1024; const memDiff = memAfter - memBefore; if (memDiff > 50) { console.log(`⚠️ Memory spike during masking: +${memDiff.toFixed(2)}MB (${memBefore.toFixed(2)}→${memAfter.toFixed(2)}MB)`); } return result; } /** * Memory-safe array masking with batch processing */ function maskArraySafe(arr: any[], config: Required): any[] { const BATCH_SIZE = 1000; // Process 1000 items at a time const totalItems = arr.length; if (totalItems <= BATCH_SIZE) { // Small array - process normally return arr.map(item => maskSensitiveData(item, config)); } console.log(`📊 Processing large array (${totalItems} items) in batches of ${BATCH_SIZE}`); const result: any[] = []; let processedItems = 0; for (let i = 0; i < totalItems; i += BATCH_SIZE) { const batch = arr.slice(i, i + BATCH_SIZE); const batchStart = Date.now(); // Process batch const maskedBatch = batch.map(item => maskSensitiveData(item, config)); result.push(...maskedBatch); processedItems += batch.length; const batchTime = Date.now() - batchStart; // Memory monitoring during batch processing const memUsage = process.memoryUsage().heapUsed / 1024 / 1024; if (memUsage > 150) { console.log(`🚨 Memory emergency brake activated during array masking at ${memUsage.toFixed(2)}MB`); console.log(`⏹️ Processed ${processedItems}/${totalItems} items before stopping`); break; } // Progress logging for large arrays if (i % (BATCH_SIZE * 10) === 0) { const progress = (processedItems / totalItems * 100).toFixed(1); console.log(`📊 Masking progress: ${progress}% (${processedItems}/${totalItems}) - Memory: ${memUsage.toFixed(2)}MB - Batch time: ${batchTime}ms`); } } console.log(`✅ Array masking completed: ${result.length}/${totalItems} items processed`); return result; } /** * Memory-safe object masking with depth limiting */ function maskObjectSafe(obj: any, config: Required, depth: number = 0): any { const MAX_DEPTH = 20; // Prevent infinite recursion const MAX_PROPERTIES = 10000; // Limit properties per object if (depth > MAX_DEPTH) { console.log(`⚠️ Maximum masking depth (${MAX_DEPTH}) reached - stopping recursion`); return '[MAX_DEPTH_REACHED]'; } const properties = Object.entries(obj); if (properties.length > MAX_PROPERTIES) { console.log(`⚠️ Large object detected (${properties.length} properties) - applying batch processing`); return maskLargeObjectInBatches(obj, config, depth); } const masked: any = {}; let processedProps = 0; for (const [key, value] of properties) { // Memory check every 100 properties if (processedProps % 100 === 0 && processedProps > 0) { const memUsage = process.memoryUsage().heapUsed / 1024 / 1024; if (memUsage > 150) { console.log(`🚨 Memory emergency brake during object masking at ${memUsage.toFixed(2)}MB`); console.log(`⏹️ Processed ${processedProps}/${properties.length} properties before stopping`); break; } } const lowerKey = key.toLowerCase(); // Check if field should be exempted if (config.exemptFields.some(field => lowerKey.includes(field.toLowerCase()) )) { masked[key] = value; processedProps++; continue; } // Check if field is sensitive const isSensitiveField = SENSITIVE_FIELD_NAMES.some(fieldName => lowerKey.includes(fieldName.toLowerCase()) ) || config.customFields.some(fieldName => lowerKey.includes(fieldName.toLowerCase()) ); if (isSensitiveField) { masked[key] = typeof value === 'string' ? maskValue(value, config) : '[MASKED]'; } else { // Recursively mask nested objects and arrays with depth tracking if (typeof value === 'object' && value !== null) { masked[key] = maskObjectSafe(value, config, depth + 1); } else if (Array.isArray(value)) { masked[key] = maskArraySafe(value, config); } else { masked[key] = maskSensitiveData(value, config); } } processedProps++; } return masked; } /** * Process large objects in batches to prevent memory overload */ function maskLargeObjectInBatches(obj: any, config: Required, depth: number): any { const PROP_BATCH_SIZE = 500; // Process 500 properties at a time const properties = Object.entries(obj); const totalProps = properties.length; console.log(`📊 Processing large object (${totalProps} properties) in batches`); const masked: any = {}; let processedProps = 0; for (let i = 0; i < totalProps; i += PROP_BATCH_SIZE) { const batchProps = properties.slice(i, i + PROP_BATCH_SIZE); const batchStart = Date.now(); // Process property batch for (const [key, value] of batchProps) { const lowerKey = key.toLowerCase(); // Quick exemption check if (config.exemptFields.some(field => lowerKey.includes(field.toLowerCase()) )) { masked[key] = value; continue; } // Sensitive field check const isSensitiveField = SENSITIVE_FIELD_NAMES.some(fieldName => lowerKey.includes(fieldName.toLowerCase()) ) || config.customFields.some(fieldName => lowerKey.includes(fieldName.toLowerCase()) ); if (isSensitiveField) { masked[key] = typeof value === 'string' ? maskValue(value, config) : '[MASKED]'; } else { // Careful recursion for nested data if (typeof value === 'object' && value !== null && depth < 15) { masked[key] = maskObjectSafe(value, config, depth + 1); } else { masked[key] = value; // Stop deep recursion } } } processedProps += batchProps.length; const batchTime = Date.now() - batchStart; // Memory monitoring const memUsage = process.memoryUsage().heapUsed / 1024 / 1024; if (memUsage > 150) { console.log(`🚨 Memory emergency brake during large object masking at ${memUsage.toFixed(2)}MB`); console.log(`⏹️ Processed ${processedProps}/${totalProps} properties before stopping`); break; } // Progress for very large objects if (processedProps % (PROP_BATCH_SIZE * 5) === 0) { const progress = (processedProps / totalProps * 100).toFixed(1); console.log(`📊 Object masking progress: ${progress}% (${processedProps}/${totalProps}) - Memory: ${memUsage.toFixed(2)}MB - Batch time: ${batchTime}ms`); } } console.log(`✅ Large object masking completed: ${processedProps}/${totalProps} properties processed`); return masked; } /** * Mask sensitive patterns in a string */ function maskString(str: string, config: Required): string { let maskedStr = str; // Apply built-in patterns if (config.maskEmails) { maskedStr = maskedStr.replace(SENSITIVE_PATTERNS.email, (match) => maskValue(match, config) ); } if (config.maskIPs) { maskedStr = maskedStr.replace(SENSITIVE_PATTERNS.ipAddress, (match) => maskValue(match, config) ); } if (config.maskConnectionStrings) { maskedStr = maskedStr.replace(SENSITIVE_PATTERNS.connectionString, (match) => maskValue(match, config) ); } // Apply credit card masking maskedStr = maskedStr.replace(SENSITIVE_PATTERNS.creditCard, (match) => maskValue(match, config) ); // Apply phone number masking maskedStr = maskedStr.replace(SENSITIVE_PATTERNS.phone, (match) => maskValue(match, config) ); // Apply SSN masking maskedStr = maskedStr.replace(SENSITIVE_PATTERNS.ssn, (match) => maskValue(match, config) ); // Apply JWT token masking maskedStr = maskedStr.replace(SENSITIVE_PATTERNS.jwt, (match) => maskValue(match, config) ); // Apply API key masking maskedStr = maskedStr.replace(SENSITIVE_PATTERNS.apiKey, (match) => maskValue(match, config) ); // Apply custom patterns Object.entries(config.customPatterns).forEach(([name, pattern]) => { maskedStr = maskedStr.replace(pattern, (match) => maskValue(match, config) ); }); return maskedStr; } /** * Mask sensitive fields in an object (Legacy function - now uses safe version) */ function maskObject(obj: any, config: Required): any { // Redirect to memory-safe version return maskObjectSafe(obj, config, 0); } /** * Mask a single value */ function maskValue(value: string, config: Required): string { if (!value) return value; const { maskingChar, preserveLength, showLastChars } = config; if (preserveLength) { if (showLastChars > 0 && value.length > showLastChars) { const maskedPart = maskingChar.repeat(value.length - showLastChars); const visiblePart = value.slice(-showLastChars); return maskedPart + visiblePart; } return maskingChar.repeat(value.length); } if (showLastChars > 0 && value.length > showLastChars) { const visiblePart = value.slice(-showLastChars); return '[MASKED]' + visiblePart; } return '[MASKED]'; } /** * Create a masking configuration for different environments */ export function createMaskingConfig(environment: 'development' | 'staging' | 'production'): MaskingConfig { const baseConfig: MaskingConfig = { enabled: true, maskingChar: '*', preserveLength: false, showLastChars: 0, maskEmails: true, maskIPs: false, maskConnectionStrings: true }; switch (environment) { case 'development': return { ...baseConfig, enabled: false, // Disable masking in development for easier debugging maskIPs: false }; case 'staging': return { ...baseConfig, enabled: true, showLastChars: 4, // Show last 4 characters for debugging maskIPs: false }; case 'production': return { ...baseConfig, enabled: true, showLastChars: 0, // Full masking in production maskIPs: true, // Mask IPs in production for privacy preserveLength: false }; default: return baseConfig; } } /** * Validate masking configuration */ export function validateMaskingConfig(config: MaskingConfig): { isValid: boolean; errors: string[] } { const errors: string[] = []; if (config.showLastChars && config.showLastChars < 0) { errors.push('showLastChars must be non-negative'); } if (config.maskingChar && config.maskingChar.length !== 1) { errors.push('maskingChar must be a single character'); } if (config.customPatterns) { Object.entries(config.customPatterns).forEach(([name, pattern]) => { if (!(pattern instanceof RegExp)) { errors.push(`Custom pattern '${name}' must be a RegExp`); } }); } return { isValid: errors.length === 0, errors }; }