import { Duration, RemovalPolicy, CustomResource } from 'aws-cdk-lib'; import { IVpc, ISecurityGroup } from 'aws-cdk-lib/aws-ec2'; import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; import { Function as LambdaFunction } from 'aws-cdk-lib/aws-lambda'; import { IDatabaseCluster, IDatabaseInstance } from 'aws-cdk-lib/aws-rds'; import { Bucket } from 'aws-cdk-lib/aws-s3'; import { BucketDeployment } from 'aws-cdk-lib/aws-s3-deployment'; import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; import { StateMachine } from 'aws-cdk-lib/aws-stepfunctions'; import { Provider } from 'aws-cdk-lib/custom-resources'; import { Construct } from 'constructs'; /** * Supported database engines */ export declare enum DatabaseEngine { MYSQL = "mysql", POSTGRESQL = "postgresql" } /** * Supported file types for data loading */ export declare enum FileType { /** Standard SQL file */ SQL = "sql", /** MySQL dump file generated by mysqldump */ MYSQLDUMP = "mysqldump", /** PostgreSQL dump file generated by pg_dump */ PGDUMP = "pgdump" } /** * Database connection configuration */ export interface DatabaseConfig { /** Database engine type */ readonly engine: DatabaseEngine; /** Database cluster (for Aurora) */ readonly cluster?: IDatabaseCluster; /** Database instance (for RDS) */ readonly instance?: IDatabaseInstance; /** Database credentials secret */ readonly secret: ISecret; /** Database name to connect to */ readonly databaseName: string; /** VPC where the database is located */ readonly vpc: IVpc; /** Security group for database access */ readonly securityGroup: ISecurityGroup; } /** * File input configuration */ export interface FileInput { /** Path to the file (local path or S3 URI) */ readonly filePath: string; /** Type of file */ readonly fileType: FileType; /** Execution order (lower numbers execute first) */ readonly executionOrder?: number; /** Whether to continue on error */ readonly continueOnError?: boolean; } /** * Properties for the DataLoader construct */ export interface DataLoaderProps { /** Database configuration */ readonly databaseConfig: DatabaseConfig; /** List of files to load */ readonly fileInputs: FileInput[]; /** Optional removal policy for resources (defaults to DESTROY) */ readonly removalPolicy?: RemovalPolicy; /** Optional timeout for Lambda function (defaults to 15 minutes) */ readonly timeout?: Duration; /** Optional memory size for Lambda function (defaults to 1024 MB) */ readonly memorySize?: number; } /** * DataLoader construct for loading data into Aurora/RDS databases * * This construct provides a simplified solution for loading data from various file formats * (SQL, mysqldump, pg_dump) into MySQL or PostgreSQL databases. It uses S3 for file storage, * Step Functions for orchestration, and Lambda for processing. * * Architecture: * 1. Files are uploaded to S3 bucket * 2. Step Function is triggered with list of S3 keys * 3. Step Function iterates over files in execution order * 4. Lambda function processes each file against the database * * Example usage: * Create a DataLoader with database configuration and file inputs. * The construct will handle uploading files to S3, creating a Step Function * to orchestrate processing, and executing the data loading pipeline. */ export declare class DataLoader extends Construct { /** The S3 bucket used for storing files */ readonly bucket: Bucket; /** The Step Functions state machine for orchestration */ readonly stateMachine: StateMachine; /** The Lambda function that processes the data loading */ readonly processorFunction: LambdaFunction; /** The bucket deployment for uploading files */ bucketDeployment?: BucketDeployment; /** The custom resource provider for triggering state machine execution */ readonly customResourceProvider: Provider; /** The custom resource that triggers the state machine */ readonly executionTrigger: CustomResource; /** The file inputs configuration */ private readonly fileInputs; constructor(scope: Construct, id: string, props: DataLoaderProps); /** * Grants additional IAM permissions to the execution trigger Lambda function * @param statement The IAM policy statement to add */ grantExecutionTriggerPermissions(statement: PolicyStatement): void; /** * Validates the construct properties * @param props The DataLoader properties * @private */ private _validateProps; /** * Creates the S3 bucket for storing files * @param removalPolicy The removal policy to apply * @returns The created S3 bucket * @private */ private _createBucket; /** * Creates the Lambda function for processing data loading * @param props The DataLoader properties * @returns The created Lambda function * @private */ private _createProcessorFunction; /** * Creates the Step Functions state machine * @returns The created state machine * @private */ private _createStateMachine; /** * Grants necessary permissions to the Lambda function * @param lambdaFunction The Lambda function * @param props The DataLoader properties * @private */ private _grantPermissions; /** * Sets up file processing by uploading files to S3 * @param props The DataLoader properties * @private */ private _setupFileProcessing; /** * Creates a custom resource provider for triggering state machine execution * @returns The custom resource provider * @private */ private _createCustomResourceProvider; /** * Creates a custom resource to trigger state machine execution * @returns The custom resource * @private */ private _createExecutionTrigger; /** * Gets the ordered file keys for execution * @returns Array of S3 keys in execution order * @private */ private _getOrderedFileKeys; }