import { ApiLogModel, getApiLogModel } from '../models/apiLog'; import { Config } from '../types/config'; import { Request, Response } from 'express'; import { maskSensitiveData } from '../lib/dataMasking'; export interface ApiLogData { request_time?: Date; response_time?: Date; method: string; path: string; requestBody?: any; requestHeaders?: any; responseStatus?: any; requestQuery?: any; requestParams?: any; client_ip?: string; client_agent?: string; responseBody?: any; } export async function saveApiLog(logData: ApiLogData, config?: Config) { try { const collectionName = config?.collections?.apiLogsCollectionName || 'apilogs'; const ApiLogModelInstance = getApiLogModel(collectionName); // Apply sensitive data masking if enabled const maskedLogData = config?.dataMasking?.enabled ? maskSensitiveData(logData, config.dataMasking) : logData; const apiLog = new ApiLogModelInstance(maskedLogData); await apiLog.save(); return apiLog; } catch (error) { console.error('Error saving API log:', error); throw error; } } export async function getApiLogs(filters?: { startDate?: Date; endDate?: Date; method?: string; path?: string; client_ip?: string; limit?: number; skip?: number; }, config?: Config) { try { const collectionName = config?.collections?.apiLogsCollectionName || 'apilogs'; const ApiLogModelInstance = getApiLogModel(collectionName); const query: any = {}; if (filters?.startDate || filters?.endDate) { query.request_time = {}; if (filters.startDate) query.request_time.$gte = filters.startDate; if (filters.endDate) query.request_time.$lte = filters.endDate; } if (filters?.method) query.method = filters.method; if (filters?.path) query.path = new RegExp(filters.path, 'i'); if (filters?.client_ip) query.client_ip = filters.client_ip; // Memory-safe approach for large datasets const requestedLimit = filters?.limit || 100; const skip = filters?.skip || 0; // First check if it's a potentially large query const totalCount = await ApiLogModelInstance.countDocuments(query); console.log(`📊 Found ${totalCount} matching logs from collection '${collectionName}'`); // Apply safety limits for large datasets let safeLimit = requestedLimit; if (totalCount > 50000) { safeLimit = Math.min(requestedLimit, 200); // Ultra safe for 50k+ records console.log(`⚠️ Large dataset detected (${totalCount} records). Limiting to ${safeLimit} for memory safety.`); } else if (totalCount > 10000) { safeLimit = Math.min(requestedLimit, 500); // Safe for 10k+ records } // Use streaming cursor for better memory management const logs: any[] = []; let processedDocs = 0; let skippedDocs = 0; const cursor = ApiLogModelInstance.find(query) .sort({ request_time: -1 }) .cursor(); await cursor.eachAsync(async (doc: any) => { // Handle skip if (skippedDocs < skip) { skippedDocs++; return true; // Continue to next document } // Memory monitoring const memUsage = process.memoryUsage(); const memUsageMB = memUsage.heapUsed / 1024 / 1024; if (memUsageMB > 100) { // Emergency brake at 100MB console.log(`🚨 Memory emergency brake activated at ${memUsageMB.toFixed(2)}MB. Stopping processing.`); return false; // Stop cursor iteration } logs.push(doc); processedDocs++; // Stop when we reach the safe limit if (processedDocs >= safeLimit) { return false; // Stop cursor iteration } // Progress monitoring for large datasets if (processedDocs % 100 === 0 && totalCount > 1000) { console.log(`📊 Processed ${processedDocs}/${safeLimit} logs. Memory: ${memUsageMB.toFixed(2)}MB`); } return true; // Continue processing }); await cursor.close(); console.log(`✅ Returned ${logs.length} logs. Memory: ${(process.memoryUsage().heapUsed / 1024 / 1024).toFixed(2)}MB`); return logs; } catch (error) { console.error('Error fetching API logs:', error); throw error; } } export function createApiLogMiddleware(config?: Config) { return (req: Request, res: Response, next: Function) => { const startTime = new Date(); // Store original res.json and res.send methods const originalJson = res.json; const originalSend = res.send; let responseBody: any; // Override res.json to capture response body res.json = function(body: any) { responseBody = body; return originalJson.call(this, body); }; // Override res.send to capture response body res.send = function(body: any) { if (!responseBody) responseBody = body; return originalSend.call(this, body); }; // Log the request when response finishes res.on('finish', async () => { const endTime = new Date(); const logData: ApiLogData = { request_time: startTime, response_time: endTime, method: req.method, path: req.path, requestBody: req.body, requestHeaders: req.headers, responseStatus: res.statusCode, requestQuery: req.query, requestParams: req.params, client_ip: req.ip || req.connection.remoteAddress || '', client_agent: req.get('User-Agent') || '', responseBody: responseBody, }; try { await saveApiLog(logData, config); } catch (error) { console.error('Failed to save API log:', error); } }); next(); }; } export async function getApiLogsByHour(date: string, hourRange: string, config?: Config) { try { const collectionName = config?.collections?.apiLogsCollectionName || 'apilogs'; const ApiLogModelInstance = getApiLogModel(collectionName); const [startHour] = hourRange.split('-').map(h => parseInt(h)); const startDate = new Date(`${date}T${startHour.toString().padStart(2, '0')}:00:00.000Z`); const endDate = new Date(startDate.getTime() + 60 * 60 * 1000); // Add 1 hour // Memory-safe streaming cursor approach for large datasets (11+ lac records) const logs: any[] = []; let processedDocs = 0; const CHUNK_SIZE = 100; // Process in small chunks // First check total count for memory protection const totalCount = await ApiLogModelInstance.countDocuments({ request_time: { $gte: startDate, $lt: endDate } }); console.log(`📊 Found ${totalCount} logs for ${date} hour ${hourRange} from collection '${collectionName}'`); // Apply dynamic limits based on dataset size let maxResults = 1000; // Default limit if (totalCount > 100000) maxResults = 500; // 1 lac+ records = ultra safe if (totalCount > 50000) maxResults = 800; // 50k+ records = very safe if (totalCount > 10000) maxResults = 1200; // 10k+ records = safe if (totalCount > maxResults) { console.log(`⚠️ Large dataset detected (${totalCount} records). Limiting to ${maxResults} for memory safety.`); } // Create streaming cursor const cursor = ApiLogModelInstance.find({ request_time: { $gte: startDate, $lt: endDate } }).sort({ request_time: 1 }).cursor(); // Stream documents one by one with memory monitoring await cursor.eachAsync(async (doc: any) => { // Memory emergency brake const memUsage = process.memoryUsage(); const memUsageMB = memUsage.heapUsed / 1024 / 1024; if (memUsageMB > 120) { // Emergency brake at 120MB console.log(`🚨 Memory emergency brake activated at ${memUsageMB.toFixed(2)}MB. Stopping processing.`); return false; // Stop cursor iteration } logs.push(doc); processedDocs++; // Limit results for memory safety if (processedDocs >= maxResults) { console.log(`📝 Reached processing limit of ${maxResults} documents for memory safety.`); return false; // Stop cursor iteration } // Memory monitoring every 50 docs if (processedDocs % 50 === 0) { console.log(`📊 Processed ${processedDocs}/${Math.min(totalCount, maxResults)} docs. Memory: ${memUsageMB.toFixed(2)}MB`); } return true; // Continue processing }); await cursor.close(); console.log(`✅ Completed processing ${logs.length} logs for ${date} hour ${hourRange}. Memory: ${(process.memoryUsage().heapUsed / 1024 / 1024).toFixed(2)}MB`); return logs; } catch (error) { console.error('Error fetching API logs by hour:', error); throw error; } }