import { promises as fsp } from 'node:fs'; import path from 'node:path'; import { promisify } from 'node:util'; import { exec as execCallback } from 'node:child_process'; import AdmZip from 'adm-zip'; import chalk from 'chalk'; import { CollectionModuleConfig } from '../../domain/collection-modules/collection-module-config'; import { CollectionModuleStaticFileName } from '../../domain/collection-modules/collection-module-file-names'; import { existsAsync } from '../fs-helpers'; import { symbols } from '../symbols'; import { bundleCodeFiles } from './read-collection-module-definition'; const exec = promisify(execCallback); // AWS Lambda size limits const LAMBDA_SIZE_LIMIT_COMPRESSED_MB = 50; const LAMBDA_SIZE_LIMIT_COMPRESSED_BYTES = LAMBDA_SIZE_LIMIT_COMPRESSED_MB * 1024 * 1024; const LAMBDA_SIZE_LIMIT_UNCOMPRESSED_MB = 250; // Warning threshold (warn at 80% of limit) const SIZE_WARNING_THRESHOLD = 0.8; // Lambda handler template // IMPORTANT: This handler is duplicated in: // - app.insurance/collection-module-definition-code/actions/create-lambda-code-zip-file.ts (indexCode template) // - service.collection-module-execution/app.js // Any changes must be applied to all locations. const LAMBDA_HANDLER_CODE = `const util = require('util'); const crypto = require('crypto'); const { DynamoDBClient } = require('@aws-sdk/client-dynamodb'); const { DynamoDBDocumentClient, BatchWriteCommand } = require('@aws-sdk/lib-dynamodb'); // Initialize DynamoDB client at module level to reuse connections across invocations const dynamoConfig = { region: process.env.AWS_REGION || 'af-south-1' }; const dynamoClient = new DynamoDBClient(dynamoConfig); const docClient = DynamoDBDocumentClient.from(dynamoClient); exports.handler = async function (event) { const logs = []; const originalMethods = {}; // Intercept console methods to collect logs const consoleMethods = ['log', 'info', 'warn', 'error', 'debug']; consoleMethods.forEach((method) => { const originalMethod = console[method]; originalMethods[method] = originalMethod; console[method] = function (...args) { const timestamp = new Date().toISOString(); const timestampMs = Date.now(); const level = method === 'log' ? 'info' : method; const stringifiedArgs = args ? args.map(arg => (typeof arg === 'string' ? arg : util.inspect(arg))) : []; logs.push({ level, timestamp, timestampMs, message: stringifiedArgs.join(' ') }); originalMethod.apply(console, args); }; }); // Extract functionName early for use in error handler let functionName; try { const { functionName: fn, functionArgs, env, loggingContext } = event; functionName = fn; Object.entries(env || {}).forEach(([key, value]) => { process.env[key] = value; }); const collectionModuleCode = require('./main'); const result = await collectionModuleCode[fn](functionArgs); Object.entries(env || {}).forEach(([key]) => { delete process.env[key]; }); // Restore console methods before writing logs consoleMethods.forEach((method) => { console[method] = originalMethods[method]; }); // Write logs to DynamoDB await writeLogsToDynamoDB(logs, loggingContext, fn, originalMethods.log); return { result }; } catch (error) { // Safely access event properties in case error occurred before destructuring const safeLoggingContext = event?.loggingContext; const safeFunctionName = functionName || event?.functionName; Object.entries(event?.env || {}).forEach(([key]) => { delete process.env[key]; }); // Restore console methods before writing logs consoleMethods.forEach((method) => { console[method] = originalMethods[method]; }); // Write logs to DynamoDB (includes error logs) await writeLogsToDynamoDB(logs, safeLoggingContext, safeFunctionName, originalMethods.log); return { error: { name: error.name, message: error.message, stack: error.stack } }; } }; // IMPORTANT: This function is duplicated in: // - service.collection-module-execution/app.js // - app.insurance/collection-module-definition-code/actions/create-lambda-code-zip-file.ts (indexCode template) // Any changes must be applied to both locations. async function writeLogsToDynamoDB(logs, loggingContext, functionName, originalConsoleLog) { if (!logs || logs.length === 0) { return; } if (!loggingContext || !loggingContext.tableName) { originalConsoleLog('[COLLECTION_MODULE_LOGS] ERROR: No loggingContext or tableName provided, skipping DynamoDB log write'); return; } try { const { tableName, collectionModuleKey, collectionModuleId, policyId, organizationId, environment } = loggingContext; // Calculate TTL (30 days from now in seconds, configurable via env var) const ttlDays = parseInt(process.env.CM_LOGS_TTL_DAYS || '30', 10); const ttl = Math.floor(Date.now() / 1000) + (ttlDays * 24 * 60 * 60); // Prepare DynamoDB items (omit null values for consistency) const items = logs.map((log, index) => { // Use crypto.randomUUID() for uniqueness when policyId is missing const uniqueId = policyId || crypto.randomUUID(); const item = { pk: \`\${collectionModuleKey}#\${environment}\`, sk: \`\${uniqueId}#\${log.timestampMs}#\${index}\`, organization_id: organizationId, environment: environment, function_name: functionName, timestamp: log.timestamp, level: log.level, message: log.message, ttl: ttl }; // Only include optional fields if they have values if (policyId) { item.policy_id = policyId; } if (collectionModuleId) { item.collection_module_id = collectionModuleId; } return item; }); // Write in batches of 25 (DynamoDB BatchWrite limit) const batchSize = 25; let batchCount = 0; for (let i = 0; i < items.length; i += batchSize) { const batch = items.slice(i, i + batchSize); const putRequests = batch.map(item => ({ PutRequest: { Item: item } })); const command = new BatchWriteCommand({ RequestItems: { [tableName]: putRequests } }); const response = await docClient.send(command); // Handle unprocessed items (throttling or capacity issues) if (response.UnprocessedItems && Object.keys(response.UnprocessedItems).length > 0) { const unprocessedCount = response.UnprocessedItems[tableName]?.length || 0; originalConsoleLog(\`[COLLECTION_MODULE_LOGS] WARNING: \${unprocessedCount} items were not processed in batch \${batchCount + 1}. They will be retried.\`, { batchNumber: batchCount + 1, unprocessedCount, tableName }); // Retry unprocessed items with exponential backoff (max 3 retries) let retryCount = 0; let remainingUnprocessed = response.UnprocessedItems[tableName] || []; while (remainingUnprocessed.length > 0 && retryCount < 3) { retryCount++; const delay = Math.min(1000 * Math.pow(2, retryCount - 1), 5000); // Exponential backoff, max 5s await new Promise(resolve => setTimeout(resolve, delay)); const retryCommand = new BatchWriteCommand({ RequestItems: { [tableName]: remainingUnprocessed } }); const retryResponse = await docClient.send(retryCommand); if (retryResponse.UnprocessedItems && retryResponse.UnprocessedItems[tableName]) { remainingUnprocessed = retryResponse.UnprocessedItems[tableName]; originalConsoleLog(\`[COLLECTION_MODULE_LOGS] Retry \${retryCount} for batch \${batchCount + 1}: \${remainingUnprocessed.length} items still unprocessed\`, { batchNumber: batchCount + 1, retryCount, unprocessedCount: remainingUnprocessed.length }); } else { remainingUnprocessed = []; } } if (remainingUnprocessed.length > 0) { originalConsoleLog(\`[COLLECTION_MODULE_LOGS] ERROR: \${remainingUnprocessed.length} items failed to write after \${retryCount} retries in batch \${batchCount + 1}\`, { batchNumber: batchCount + 1, failedCount: remainingUnprocessed.length }); } } batchCount++; } originalConsoleLog(\`[COLLECTION_MODULE_LOGS] Successfully wrote \${logs.length} logs in \${batchCount} batches to \${tableName}\`); } catch (error) { // Log to CloudWatch but don't fail the Lambda execution originalConsoleLog('[COLLECTION_MODULE_LOGS] ERROR: Failed to write logs to DynamoDB', { error: error.message, errorName: error.name, stack: error.stack }); } } `; /** * Calculate the uncompressed size of a directory */ const getDirectorySize = async (dirPath: string): Promise => { let totalSize = 0; const calculateSize = async (currentPath: string): Promise => { const stats = await fsp.stat(currentPath); if (stats.isFile()) { totalSize += stats.size; } else if (stats.isDirectory()) { const entries = await fsp.readdir(currentPath); await Promise.all(entries.map((entry) => calculateSize(path.join(currentPath, entry)))); } }; await calculateSize(dirPath); return totalSize; }; /** * Check Lambda package size and warn if it exceeds or approaches limits * Also checks for potential zip bombs by validating compression ratio */ const checkPackageSize = (zipBuffer: Buffer, uncompressedSize: number): void => { const compressedSizeMB = zipBuffer.length / (1024 * 1024); const compressedSizeKB = zipBuffer.length / 1024; const uncompressedSizeMB = uncompressedSize / (1024 * 1024); const compressionRatio = uncompressedSize / zipBuffer.length; // Display size info console.log( chalk.blue( `Created deployment package (${compressedSizeKB.toFixed(2)} KB compressed, ${uncompressedSizeMB.toFixed( 2, )} MB uncompressed)`, ), ); // Check for suspicious compression ratio (potential zip bomb) if (compressionRatio > 10) { console.error(chalk.red(`\n${symbols.error}ERROR: Suspicious compression ratio detected (${compressionRatio.toFixed(1)}x)!`)); console.error(chalk.red('This could indicate a zip bomb or highly unusual package contents.')); console.error(chalk.red('Deployment aborted for security reasons.\n')); throw new Error('Suspicious compression ratio detected - potential zip bomb'); } // Check if uncompressed size exceeds limit if (uncompressedSizeMB > LAMBDA_SIZE_LIMIT_UNCOMPRESSED_MB) { console.error( chalk.red( `\n${symbols.error}ERROR: Uncompressed package size (${uncompressedSizeMB.toFixed( 2, )} MB) exceeds the uncompressed size limit of ${LAMBDA_SIZE_LIMIT_UNCOMPRESSED_MB} MB!`, ), ); console.error(chalk.red('The deployment will fail.\n')); console.error(chalk.yellow('Suggestions to reduce package size:')); console.error(chalk.yellow(' 1. Remove unnecessary dependencies from package.json')); console.error(chalk.yellow(' 2. Use smaller alternative packages')); console.error(chalk.yellow(' 3. Contact Root Platform team to add frequently-used deps to base layer')); throw new Error('Package size exceeds uncompressed limit'); } // Check if compressed size exceeds limit if (zipBuffer.length > LAMBDA_SIZE_LIMIT_COMPRESSED_BYTES) { console.error( chalk.red( `\n${symbols.error}ERROR: Package size (${compressedSizeMB.toFixed( 2, )} MB) exceeds the compressed size limit of ${LAMBDA_SIZE_LIMIT_COMPRESSED_MB} MB!`, ), ); console.error(chalk.red('The deployment will likely fail.\n')); console.error(chalk.yellow('Suggestions to reduce package size:')); console.error(chalk.yellow(' 1. Remove unnecessary dependencies from package.json')); console.error(chalk.yellow(' 2. Use smaller alternative packages')); console.error(chalk.yellow(' 3. Contact Root Platform team to add frequently-used deps to base layer')); throw new Error('Package size exceeds compressed limit'); } // Check if approaching the compressed limit (80% threshold) const percentOfLimit = (compressedSizeMB / LAMBDA_SIZE_LIMIT_COMPRESSED_MB) * 100; if (compressedSizeMB > LAMBDA_SIZE_LIMIT_COMPRESSED_MB * SIZE_WARNING_THRESHOLD) { console.warn( chalk.yellow( `\n${symbols.warning}Warning: Package size is ${percentOfLimit.toFixed( 1, )}% of the ${LAMBDA_SIZE_LIMIT_COMPRESSED_MB} MB compressed limit`, ), ); console.warn( chalk.yellow( `Consider optimizing dependencies to stay well below the limit (${compressedSizeMB.toFixed( 2, )} MB / ${LAMBDA_SIZE_LIMIT_COMPRESSED_MB} MB).\n`, ), ); } // Check if approaching the uncompressed limit (80% threshold) const percentOfUncompressedLimit = (uncompressedSizeMB / LAMBDA_SIZE_LIMIT_UNCOMPRESSED_MB) * 100; if (uncompressedSizeMB > LAMBDA_SIZE_LIMIT_UNCOMPRESSED_MB * SIZE_WARNING_THRESHOLD) { console.warn( chalk.yellow( `\n${symbols.warning}Warning: Uncompressed size is ${percentOfUncompressedLimit.toFixed( 1, )}% of the ${LAMBDA_SIZE_LIMIT_UNCOMPRESSED_MB} MB uncompressed limit`, ), ); console.warn( chalk.yellow( `Consider optimizing dependencies (${uncompressedSizeMB.toFixed( 2, )} MB / ${LAMBDA_SIZE_LIMIT_UNCOMPRESSED_MB} MB).\n`, ), ); } }; /** * Create a ZIP file from a temp directory with Lambda package structure */ const createZipFromTempDirectory = async (tempDir: string): Promise<{ buffer: Buffer; uncompressedSize: number }> => { // Calculate uncompressed size first for zip bomb detection const uncompressedSize = await getDirectorySize(tempDir); const zip = new AdmZip(); // Add all files from temp directory to root of ZIP // This creates structure: index.js, main.js, package.json, node_modules/ at root const entries = await fsp.readdir(tempDir, { withFileTypes: true }); for (const entry of entries) { const entryPath = path.join(tempDir, entry.name); if (entry.isDirectory()) { // Add directory (e.g., node_modules) with its name zip.addLocalFolder(entryPath, entry.name); } else { // Add file (e.g., index.js, main.js, package.json) to root zip.addLocalFile(entryPath); } } const buffer = zip.toBuffer(); return { buffer, uncompressedSize }; }; /** * Create a complete Lambda deployment package * Returns a Buffer containing the ZIP file with: * - index.js (Lambda handler) * - main.js (esbuild-bundled user code) * - package.json (user dependencies) * - node_modules/ (installed dependencies) */ export const createLambdaPackage = async (params: { directory: string; collectionModuleConfig: CollectionModuleConfig; }): Promise => { const { directory, collectionModuleConfig } = params; const codeDirectory = path.join(directory, 'code'); const packageJsonPath = path.join(directory, CollectionModuleStaticFileName.PackageJson); // Create a temporary directory for the Lambda package const tempDir = path.join(directory, '.tmp-lambda-package'); await fsp.mkdir(tempDir, { recursive: true }); try { console.log(chalk.blue('\nCreating deployment package...')); // 1. Create Lambda handler (index.js) await fsp.writeFile(path.join(tempDir, 'index.js'), LAMBDA_HANDLER_CODE); // 2. Bundle user code with esbuild and save as main.js console.log('Bundling collection module code...'); const bundledCode = await bundleCodeFiles({ directory: codeDirectory, collectionModuleConfig, }); await fsp.writeFile(path.join(tempDir, 'main.js'), bundledCode); // 3. Copy package.json if it exists if (await existsAsync(packageJsonPath)) { const packageJsonContent = await fsp.readFile(packageJsonPath, 'utf8'); await fsp.writeFile(path.join(tempDir, 'package.json'), packageJsonContent); // 4. Run npm install to create node_modules console.log('Installing dependencies...'); const { stderr } = await exec('npm install --production --no-package-lock', { cwd: tempDir, }); if (stderr && !stderr.includes('WARN')) { console.warn('npm install warnings:', stderr); } } else { console.log(chalk.gray('No package.json found - skipping dependency installation')); } // 5. Create ZIP with all files at root level const { buffer: zipBuffer, uncompressedSize } = await createZipFromTempDirectory(tempDir); // 6. Validate package size (includes zip bomb detection) checkPackageSize(zipBuffer, uncompressedSize); // 7. Clean up temp directory await fsp.rm(tempDir, { recursive: true, force: true }); return zipBuffer; } catch (error) { // Ensure cleanup even on error await fsp.rm(tempDir, { recursive: true, force: true }); throw error; } };