{ "author": { "name": "Amazon Web Services", "organization": true, "roles": [ "author" ], "url": "https://aws.amazon.com" }, "bundled": { "@types/js-yaml": "^4.0.9", "js-yaml": "^4.1.0", "semver": "^7.5.4", "simple-base": "^1.0.0" }, "dependencies": { "@aws-cdk/lambda-layer-kubectl-v27": "^2.0.0", "aws-cdk-lib": "^2.114.1", "constructs": "^10.3.0" }, "dependencyClosure": { "@aws-cdk/asset-awscli-v1": { "targets": { "dotnet": { "namespace": "Amazon.CDK.Asset.AwsCliV1", "packageId": "Amazon.CDK.Asset.AwsCliV1" }, "go": { "moduleName": "github.com/cdklabs/awscdk-asset-awscli-go", "packageName": "awscliv1" }, "java": { "maven": { "artifactId": "cdk-asset-awscli-v1", "groupId": "software.amazon.awscdk" }, "package": "software.amazon.awscdk.cdk.asset.awscli.v1" }, "js": { "npm": "@aws-cdk/asset-awscli-v1" }, "python": { "distName": "aws-cdk.asset-awscli-v1", "module": "aws_cdk.asset_awscli_v1" } } }, "@aws-cdk/asset-kubectl-v20": { "targets": { "dotnet": { "namespace": "Amazon.CDK.Asset.KubectlV20", "packageId": "Amazon.CDK.Asset.KubectlV20" }, "go": { "moduleName": "github.com/cdklabs/awscdk-asset-kubectl-go", "packageName": "kubectlv20" }, "java": { "maven": { "artifactId": "cdk-asset-kubectl-v20", "groupId": "software.amazon.awscdk" }, "package": "software.amazon.awscdk.cdk.asset.kubectl.v20" }, "js": { "npm": "@aws-cdk/asset-kubectl-v20" }, "python": { "distName": "aws-cdk.asset-kubectl-v20", "module": "aws_cdk.asset_kubectl_v20" } } }, "@aws-cdk/asset-node-proxy-agent-v6": { "targets": { "dotnet": { "namespace": "Amazon.CDK.Asset.NodeProxyAgentV6", "packageId": "Amazon.CDK.Asset.NodeProxyAgentV6" }, "go": { "moduleName": "github.com/cdklabs/awscdk-asset-node-proxy-agent-go", "packageName": "nodeproxyagentv6" }, "java": { "maven": { "artifactId": "cdk-asset-node-proxy-agent-v6", "groupId": "software.amazon.awscdk" }, "package": "software.amazon.awscdk.cdk.asset.node.proxy.agent.v6" }, "js": { "npm": "@aws-cdk/asset-node-proxy-agent-v6" }, "python": { "distName": "aws-cdk.asset-node-proxy-agent-v6", "module": "aws_cdk.asset_node_proxy_agent_v6" } } }, "@aws-cdk/lambda-layer-kubectl-v27": { "targets": { "dotnet": { "namespace": "Amazon.CDK.LambdaLayer.KubectlV27", "packageId": "Amazon.CDK.LambdaLayer.KubectlV27" }, "go": { "moduleName": "github.com/cdklabs/awscdk-kubectl-go", "packageName": "kubectlv27" }, "java": { "maven": { "artifactId": "cdk-lambda-layer-kubectl-v27", "groupId": "software.amazon.awscdk" }, "package": "software.amazon.awscdk.cdk.lambdalayer.kubectl.v27" }, "js": { "npm": "@aws-cdk/lambda-layer-kubectl-v27" }, "python": { "distName": "aws-cdk.lambda-layer-kubectl-v27", "module": "aws_cdk.lambda_layer_kubectl_v27" } } }, "aws-cdk-lib": { "submodules": { "aws-cdk-lib.alexa_ask": { "targets": { "dotnet": { "namespace": "Amazon.CDK.Alexa.Ask" }, "java": { "package": "software.amazon.awscdk.alexa.ask" }, "python": { "module": "aws_cdk.alexa_ask" } } }, "aws-cdk-lib.assertions": { "targets": { "dotnet": { "namespace": "Amazon.CDK.Assertions" }, "java": { "package": "software.amazon.awscdk.assertions" }, "python": { "module": "aws_cdk.assertions" } } }, "aws-cdk-lib.assets": { "targets": { "dotnet": { "namespace": "Amazon.CDK.Assets" }, "java": { "package": "software.amazon.awscdk.assets" }, "python": { "module": "aws_cdk.assets" } } }, "aws-cdk-lib.aws_accessanalyzer": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AccessAnalyzer" }, "java": { "package": "software.amazon.awscdk.services.accessanalyzer" }, "python": { "module": "aws_cdk.aws_accessanalyzer" } } }, "aws-cdk-lib.aws_acmpca": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ACMPCA" }, "java": { "package": "software.amazon.awscdk.services.acmpca" }, "python": { "module": "aws_cdk.aws_acmpca" } } }, "aws-cdk-lib.aws_amazonmq": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AmazonMQ" }, "java": { "package": "software.amazon.awscdk.services.amazonmq" }, "python": { "module": "aws_cdk.aws_amazonmq" } } }, "aws-cdk-lib.aws_amplify": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Amplify" }, "java": { "package": "software.amazon.awscdk.services.amplify" }, "python": { "module": "aws_cdk.aws_amplify" } } }, "aws-cdk-lib.aws_amplifyuibuilder": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AmplifyUIBuilder" }, "java": { "package": "software.amazon.awscdk.services.amplifyuibuilder" }, "python": { "module": "aws_cdk.aws_amplifyuibuilder" } } }, "aws-cdk-lib.aws_apigateway": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.APIGateway" }, "java": { "package": "software.amazon.awscdk.services.apigateway" }, "python": { "module": "aws_cdk.aws_apigateway" } } }, "aws-cdk-lib.aws_apigatewayv2": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Apigatewayv2" }, "java": { "package": "software.amazon.awscdk.services.apigatewayv2" }, "python": { "module": "aws_cdk.aws_apigatewayv2" } } }, "aws-cdk-lib.aws_apigatewayv2_authorizers": {}, "aws-cdk-lib.aws_apigatewayv2_integrations": {}, "aws-cdk-lib.aws_appconfig": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AppConfig" }, "java": { "package": "software.amazon.awscdk.services.appconfig" }, "python": { "module": "aws_cdk.aws_appconfig" } } }, "aws-cdk-lib.aws_appflow": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AppFlow" }, "java": { "package": "software.amazon.awscdk.services.appflow" }, "python": { "module": "aws_cdk.aws_appflow" } } }, "aws-cdk-lib.aws_appintegrations": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AppIntegrations" }, "java": { "package": "software.amazon.awscdk.services.appintegrations" }, "python": { "module": "aws_cdk.aws_appintegrations" } } }, "aws-cdk-lib.aws_applicationautoscaling": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ApplicationAutoScaling" }, "java": { "package": "software.amazon.awscdk.services.applicationautoscaling" }, "python": { "module": "aws_cdk.aws_applicationautoscaling" } } }, "aws-cdk-lib.aws_applicationinsights": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ApplicationInsights" }, "java": { "package": "software.amazon.awscdk.services.applicationinsights" }, "python": { "module": "aws_cdk.aws_applicationinsights" } } }, "aws-cdk-lib.aws_appmesh": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AppMesh" }, "java": { "package": "software.amazon.awscdk.services.appmesh" }, "python": { "module": "aws_cdk.aws_appmesh" } } }, "aws-cdk-lib.aws_apprunner": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AppRunner" }, "java": { "package": "software.amazon.awscdk.services.apprunner" }, "python": { "module": "aws_cdk.aws_apprunner" } } }, "aws-cdk-lib.aws_appstream": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AppStream" }, "java": { "package": "software.amazon.awscdk.services.appstream" }, "python": { "module": "aws_cdk.aws_appstream" } } }, "aws-cdk-lib.aws_appsync": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AppSync" }, "java": { "package": "software.amazon.awscdk.services.appsync" }, "python": { "module": "aws_cdk.aws_appsync" } } }, "aws-cdk-lib.aws_aps": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.APS" }, "java": { "package": "software.amazon.awscdk.services.aps" }, "python": { "module": "aws_cdk.aws_aps" } } }, "aws-cdk-lib.aws_arczonalshift": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.ARCZonalShift" }, "java": { "package": "software.amazon.awscdk.services.arczonalshift" }, "python": { "module": "aws_cdk.aws_arczonalshift" } } }, "aws-cdk-lib.aws_athena": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Athena" }, "java": { "package": "software.amazon.awscdk.services.athena" }, "python": { "module": "aws_cdk.aws_athena" } } }, "aws-cdk-lib.aws_auditmanager": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AuditManager" }, "java": { "package": "software.amazon.awscdk.services.auditmanager" }, "python": { "module": "aws_cdk.aws_auditmanager" } } }, "aws-cdk-lib.aws_autoscaling": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AutoScaling" }, "java": { "package": "software.amazon.awscdk.services.autoscaling" }, "python": { "module": "aws_cdk.aws_autoscaling" } } }, "aws-cdk-lib.aws_autoscaling_common": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AutoScaling.Common" }, "java": { "package": "software.amazon.awscdk.services.autoscaling.common" }, "python": { "module": "aws_cdk.aws_autoscaling_common" } } }, "aws-cdk-lib.aws_autoscaling_hooktargets": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AutoScaling.HookTargets" }, "java": { "package": "software.amazon.awscdk.services.autoscaling.hooktargets" }, "python": { "module": "aws_cdk.aws_autoscaling_hooktargets" } } }, "aws-cdk-lib.aws_autoscalingplans": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.AutoScalingPlans" }, "java": { "package": "software.amazon.awscdk.services.autoscalingplans" }, "python": { "module": "aws_cdk.aws_autoscalingplans" } } }, "aws-cdk-lib.aws_backup": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Backup" }, "java": { "package": "software.amazon.awscdk.services.backup" }, "python": { "module": "aws_cdk.aws_backup" } } }, "aws-cdk-lib.aws_backupgateway": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.BackupGateway" }, "java": { "package": "software.amazon.awscdk.services.backupgateway" }, "python": { "module": "aws_cdk.aws_backupgateway" } } }, "aws-cdk-lib.aws_batch": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Batch" }, "java": { "package": "software.amazon.awscdk.services.batch" }, "python": { "module": "aws_cdk.aws_batch" } } }, "aws-cdk-lib.aws_billingconductor": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.BillingConductor" }, "java": { "package": "software.amazon.awscdk.services.billingconductor" }, "python": { "module": "aws_cdk.aws_billingconductor" } } }, "aws-cdk-lib.aws_budgets": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Budgets" }, "java": { "package": "software.amazon.awscdk.services.budgets" }, "python": { "module": "aws_cdk.aws_budgets" } } }, "aws-cdk-lib.aws_cassandra": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Cassandra" }, "java": { "package": "software.amazon.awscdk.services.cassandra" }, "python": { "module": "aws_cdk.aws_cassandra" } } }, "aws-cdk-lib.aws_ce": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CE" }, "java": { "package": "software.amazon.awscdk.services.ce" }, "python": { "module": "aws_cdk.aws_ce" } } }, "aws-cdk-lib.aws_certificatemanager": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CertificateManager" }, "java": { "package": "software.amazon.awscdk.services.certificatemanager" }, "python": { "module": "aws_cdk.aws_certificatemanager" } } }, "aws-cdk-lib.aws_chatbot": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Chatbot" }, "java": { "package": "software.amazon.awscdk.services.chatbot" }, "python": { "module": "aws_cdk.aws_chatbot" } } }, "aws-cdk-lib.aws_cleanrooms": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.CleanRooms" }, "java": { "package": "software.amazon.awscdk.services.cleanrooms" }, "python": { "module": "aws_cdk.aws_cleanrooms" } } }, "aws-cdk-lib.aws_cloud9": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Cloud9" }, "java": { "package": "software.amazon.awscdk.services.cloud9" }, "python": { "module": "aws_cdk.aws_cloud9" } } }, "aws-cdk-lib.aws_cloudformation": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CloudFormation" }, "java": { "package": "software.amazon.awscdk.services.cloudformation" }, "python": { "module": "aws_cdk.aws_cloudformation" } } }, "aws-cdk-lib.aws_cloudfront": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CloudFront" }, "java": { "package": "software.amazon.awscdk.services.cloudfront" }, "python": { "module": "aws_cdk.aws_cloudfront" } } }, "aws-cdk-lib.aws_cloudfront.experimental": {}, "aws-cdk-lib.aws_cloudfront_origins": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CloudFront.Origins" }, "java": { "package": "software.amazon.awscdk.services.cloudfront.origins" }, "python": { "module": "aws_cdk.aws_cloudfront_origins" } } }, "aws-cdk-lib.aws_cloudtrail": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CloudTrail" }, "java": { "package": "software.amazon.awscdk.services.cloudtrail" }, "python": { "module": "aws_cdk.aws_cloudtrail" } } }, "aws-cdk-lib.aws_cloudwatch": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CloudWatch" }, "java": { "package": "software.amazon.awscdk.services.cloudwatch" }, "python": { "module": "aws_cdk.aws_cloudwatch" } } }, "aws-cdk-lib.aws_cloudwatch_actions": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CloudWatch.Actions" }, "java": { "package": "software.amazon.awscdk.services.cloudwatch.actions" }, "python": { "module": "aws_cdk.aws_cloudwatch_actions" } } }, "aws-cdk-lib.aws_codeartifact": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodeArtifact" }, "java": { "package": "software.amazon.awscdk.services.codeartifact" }, "python": { "module": "aws_cdk.aws_codeartifact" } } }, "aws-cdk-lib.aws_codebuild": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodeBuild" }, "java": { "package": "software.amazon.awscdk.services.codebuild" }, "python": { "module": "aws_cdk.aws_codebuild" } } }, "aws-cdk-lib.aws_codecommit": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodeCommit" }, "java": { "package": "software.amazon.awscdk.services.codecommit" }, "python": { "module": "aws_cdk.aws_codecommit" } } }, "aws-cdk-lib.aws_codedeploy": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodeDeploy" }, "java": { "package": "software.amazon.awscdk.services.codedeploy" }, "python": { "module": "aws_cdk.aws_codedeploy" } } }, "aws-cdk-lib.aws_codeguruprofiler": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodeGuruProfiler" }, "java": { "package": "software.amazon.awscdk.services.codeguruprofiler" }, "python": { "module": "aws_cdk.aws_codeguruprofiler" } } }, "aws-cdk-lib.aws_codegurureviewer": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodeGuruReviewer" }, "java": { "package": "software.amazon.awscdk.services.codegurureviewer" }, "python": { "module": "aws_cdk.aws_codegurureviewer" } } }, "aws-cdk-lib.aws_codepipeline": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodePipeline" }, "java": { "package": "software.amazon.awscdk.services.codepipeline" }, "python": { "module": "aws_cdk.aws_codepipeline" } } }, "aws-cdk-lib.aws_codepipeline_actions": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodePipeline.Actions" }, "java": { "package": "software.amazon.awscdk.services.codepipeline.actions" }, "python": { "module": "aws_cdk.aws_codepipeline_actions" } } }, "aws-cdk-lib.aws_codestar": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Codestar" }, "java": { "package": "software.amazon.awscdk.services.codestar" }, "python": { "module": "aws_cdk.aws_codestar" } } }, "aws-cdk-lib.aws_codestarconnections": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodeStarConnections" }, "java": { "package": "software.amazon.awscdk.services.codestarconnections" }, "python": { "module": "aws_cdk.aws_codestarconnections" } } }, "aws-cdk-lib.aws_codestarnotifications": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CodeStarNotifications" }, "java": { "package": "software.amazon.awscdk.services.codestarnotifications" }, "python": { "module": "aws_cdk.aws_codestarnotifications" } } }, "aws-cdk-lib.aws_cognito": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Cognito" }, "java": { "package": "software.amazon.awscdk.services.cognito" }, "python": { "module": "aws_cdk.aws_cognito" } } }, "aws-cdk-lib.aws_comprehend": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Comprehend" }, "java": { "package": "software.amazon.awscdk.services.comprehend" }, "python": { "module": "aws_cdk.aws_comprehend" } } }, "aws-cdk-lib.aws_config": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Config" }, "java": { "package": "software.amazon.awscdk.services.config" }, "python": { "module": "aws_cdk.aws_config" } } }, "aws-cdk-lib.aws_connect": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Connect" }, "java": { "package": "software.amazon.awscdk.services.connect" }, "python": { "module": "aws_cdk.aws_connect" } } }, "aws-cdk-lib.aws_connectcampaigns": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ConnectCampaigns" }, "java": { "package": "software.amazon.awscdk.services.connectcampaigns" }, "python": { "module": "aws_cdk.aws_connectcampaigns" } } }, "aws-cdk-lib.aws_controltower": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ControlTower" }, "java": { "package": "software.amazon.awscdk.services.controltower" }, "python": { "module": "aws_cdk.aws_controltower" } } }, "aws-cdk-lib.aws_cur": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CUR" }, "java": { "package": "software.amazon.awscdk.services.cur" }, "python": { "module": "aws_cdk.aws_cur" } } }, "aws-cdk-lib.aws_customerprofiles": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.CustomerProfiles" }, "java": { "package": "software.amazon.awscdk.services.customerprofiles" }, "python": { "module": "aws_cdk.aws_customerprofiles" } } }, "aws-cdk-lib.aws_databrew": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DataBrew" }, "java": { "package": "software.amazon.awscdk.services.databrew" }, "python": { "module": "aws_cdk.aws_databrew" } } }, "aws-cdk-lib.aws_datapipeline": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DataPipeline" }, "java": { "package": "software.amazon.awscdk.services.datapipeline" }, "python": { "module": "aws_cdk.aws_datapipeline" } } }, "aws-cdk-lib.aws_datasync": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DataSync" }, "java": { "package": "software.amazon.awscdk.services.datasync" }, "python": { "module": "aws_cdk.aws_datasync" } } }, "aws-cdk-lib.aws_dax": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DAX" }, "java": { "package": "software.amazon.awscdk.services.dax" }, "python": { "module": "aws_cdk.aws_dax" } } }, "aws-cdk-lib.aws_detective": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Detective" }, "java": { "package": "software.amazon.awscdk.services.detective" }, "python": { "module": "aws_cdk.aws_detective" } } }, "aws-cdk-lib.aws_devicefarm": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DeviceFarm" }, "java": { "package": "software.amazon.awscdk.services.devicefarm" }, "python": { "module": "aws_cdk.aws_devicefarm" } } }, "aws-cdk-lib.aws_devopsguru": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DevOpsGuru" }, "java": { "package": "software.amazon.awscdk.services.devopsguru" }, "python": { "module": "aws_cdk.aws_devopsguru" } } }, "aws-cdk-lib.aws_directoryservice": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DirectoryService" }, "java": { "package": "software.amazon.awscdk.services.directoryservice" }, "python": { "module": "aws_cdk.aws_directoryservice" } } }, "aws-cdk-lib.aws_dlm": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DLM" }, "java": { "package": "software.amazon.awscdk.services.dlm" }, "python": { "module": "aws_cdk.aws_dlm" } } }, "aws-cdk-lib.aws_dms": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DMS" }, "java": { "package": "software.amazon.awscdk.services.dms" }, "python": { "module": "aws_cdk.aws_dms" } } }, "aws-cdk-lib.aws_docdb": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DocDB" }, "java": { "package": "software.amazon.awscdk.services.docdb" }, "python": { "module": "aws_cdk.aws_docdb" } } }, "aws-cdk-lib.aws_docdbelastic": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DocDBElastic" }, "java": { "package": "software.amazon.awscdk.services.docdbelastic" }, "python": { "module": "aws_cdk.aws_docdbelastic" } } }, "aws-cdk-lib.aws_dynamodb": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.DynamoDB" }, "java": { "package": "software.amazon.awscdk.services.dynamodb" }, "python": { "module": "aws_cdk.aws_dynamodb" } } }, "aws-cdk-lib.aws_ec2": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.EC2" }, "java": { "package": "software.amazon.awscdk.services.ec2" }, "python": { "module": "aws_cdk.aws_ec2" } } }, "aws-cdk-lib.aws_ecr": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ECR" }, "java": { "package": "software.amazon.awscdk.services.ecr" }, "python": { "module": "aws_cdk.aws_ecr" } } }, "aws-cdk-lib.aws_ecr_assets": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Ecr.Assets" }, "java": { "package": "software.amazon.awscdk.services.ecr.assets" }, "python": { "module": "aws_cdk.aws_ecr_assets" } } }, "aws-cdk-lib.aws_ecs": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ECS" }, "java": { "package": "software.amazon.awscdk.services.ecs" }, "python": { "module": "aws_cdk.aws_ecs" } } }, "aws-cdk-lib.aws_ecs_patterns": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ECS.Patterns" }, "java": { "package": "software.amazon.awscdk.services.ecs.patterns" }, "python": { "module": "aws_cdk.aws_ecs_patterns" } } }, "aws-cdk-lib.aws_efs": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.EFS" }, "java": { "package": "software.amazon.awscdk.services.efs" }, "python": { "module": "aws_cdk.aws_efs" } } }, "aws-cdk-lib.aws_eks": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.EKS" }, "java": { "package": "software.amazon.awscdk.services.eks" }, "python": { "module": "aws_cdk.aws_eks" } } }, "aws-cdk-lib.aws_elasticache": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ElastiCache" }, "java": { "package": "software.amazon.awscdk.services.elasticache" }, "python": { "module": "aws_cdk.aws_elasticache" } } }, "aws-cdk-lib.aws_elasticbeanstalk": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ElasticBeanstalk" }, "java": { "package": "software.amazon.awscdk.services.elasticbeanstalk" }, "python": { "module": "aws_cdk.aws_elasticbeanstalk" } } }, "aws-cdk-lib.aws_elasticloadbalancing": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ElasticLoadBalancing" }, "java": { "package": "software.amazon.awscdk.services.elasticloadbalancing" }, "python": { "module": "aws_cdk.aws_elasticloadbalancing" } } }, "aws-cdk-lib.aws_elasticloadbalancingv2": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ElasticLoadBalancingV2" }, "java": { "package": "software.amazon.awscdk.services.elasticloadbalancingv2" }, "python": { "module": "aws_cdk.aws_elasticloadbalancingv2" } } }, "aws-cdk-lib.aws_elasticloadbalancingv2_actions": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ElasticLoadBalancingV2.Actions" }, "java": { "package": "software.amazon.awscdk.services.elasticloadbalancingv2.actions" }, "python": { "module": "aws_cdk.aws_elasticloadbalancingv2_actions" } } }, "aws-cdk-lib.aws_elasticloadbalancingv2_targets": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ElasticLoadBalancingV2.Targets" }, "java": { "package": "software.amazon.awscdk.services.elasticloadbalancingv2.targets" }, "python": { "module": "aws_cdk.aws_elasticloadbalancingv2_targets" } } }, "aws-cdk-lib.aws_elasticsearch": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Elasticsearch" }, "java": { "package": "software.amazon.awscdk.services.elasticsearch" }, "python": { "module": "aws_cdk.aws_elasticsearch" } } }, "aws-cdk-lib.aws_emr": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.EMR" }, "java": { "package": "software.amazon.awscdk.services.emr" }, "python": { "module": "aws_cdk.aws_emr" } } }, "aws-cdk-lib.aws_emrcontainers": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.EMRContainers" }, "java": { "package": "software.amazon.awscdk.services.emrcontainers" }, "python": { "module": "aws_cdk.aws_emrcontainers" } } }, "aws-cdk-lib.aws_emrserverless": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.EMRServerless" }, "java": { "package": "software.amazon.awscdk.services.emrserverless" }, "python": { "module": "aws_cdk.aws_emrserverless" } } }, "aws-cdk-lib.aws_entityresolution": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.EntityResolution" }, "java": { "package": "software.amazon.awscdk.services.entityresolution" }, "python": { "module": "aws_cdk.aws_entityresolution" } } }, "aws-cdk-lib.aws_events": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Events" }, "java": { "package": "software.amazon.awscdk.services.events" }, "python": { "module": "aws_cdk.aws_events" } } }, "aws-cdk-lib.aws_events_targets": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Events.Targets" }, "java": { "package": "software.amazon.awscdk.services.events.targets" }, "python": { "module": "aws_cdk.aws_events_targets" } } }, "aws-cdk-lib.aws_eventschemas": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.EventSchemas" }, "java": { "package": "software.amazon.awscdk.services.eventschemas" }, "python": { "module": "aws_cdk.aws_eventschemas" } } }, "aws-cdk-lib.aws_evidently": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Evidently" }, "java": { "package": "software.amazon.awscdk.services.evidently" }, "python": { "module": "aws_cdk.aws_evidently" } } }, "aws-cdk-lib.aws_finspace": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.FinSpace" }, "java": { "package": "software.amazon.awscdk.services.finspace" }, "python": { "module": "aws_cdk.aws_finspace" } } }, "aws-cdk-lib.aws_fis": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.FIS" }, "java": { "package": "software.amazon.awscdk.services.fis" }, "python": { "module": "aws_cdk.aws_fis" } } }, "aws-cdk-lib.aws_fms": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.FMS" }, "java": { "package": "software.amazon.awscdk.services.fms" }, "python": { "module": "aws_cdk.aws_fms" } } }, "aws-cdk-lib.aws_forecast": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Forecast" }, "java": { "package": "software.amazon.awscdk.services.forecast" }, "python": { "module": "aws_cdk.aws_forecast" } } }, "aws-cdk-lib.aws_frauddetector": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.FraudDetector" }, "java": { "package": "software.amazon.awscdk.services.frauddetector" }, "python": { "module": "aws_cdk.aws_frauddetector" } } }, "aws-cdk-lib.aws_fsx": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.FSx" }, "java": { "package": "software.amazon.awscdk.services.fsx" }, "python": { "module": "aws_cdk.aws_fsx" } } }, "aws-cdk-lib.aws_gamelift": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.GameLift" }, "java": { "package": "software.amazon.awscdk.services.gamelift" }, "python": { "module": "aws_cdk.aws_gamelift" } } }, "aws-cdk-lib.aws_globalaccelerator": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.GlobalAccelerator" }, "java": { "package": "software.amazon.awscdk.services.globalaccelerator" }, "python": { "module": "aws_cdk.aws_globalaccelerator" } } }, "aws-cdk-lib.aws_globalaccelerator_endpoints": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.GlobalAccelerator.Endpoints" }, "java": { "package": "software.amazon.awscdk.services.globalaccelerator.endpoints" }, "python": { "module": "aws_cdk.aws_globalaccelerator_endpoints" } } }, "aws-cdk-lib.aws_glue": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Glue" }, "java": { "package": "software.amazon.awscdk.services.glue" }, "python": { "module": "aws_cdk.aws_glue" } } }, "aws-cdk-lib.aws_grafana": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Grafana" }, "java": { "package": "software.amazon.awscdk.services.grafana" }, "python": { "module": "aws_cdk.aws_grafana" } } }, "aws-cdk-lib.aws_greengrass": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Greengrass" }, "java": { "package": "software.amazon.awscdk.services.greengrass" }, "python": { "module": "aws_cdk.aws_greengrass" } } }, "aws-cdk-lib.aws_greengrassv2": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.GreengrassV2" }, "java": { "package": "software.amazon.awscdk.services.greengrassv2" }, "python": { "module": "aws_cdk.aws_greengrassv2" } } }, "aws-cdk-lib.aws_groundstation": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.GroundStation" }, "java": { "package": "software.amazon.awscdk.services.groundstation" }, "python": { "module": "aws_cdk.aws_groundstation" } } }, "aws-cdk-lib.aws_guardduty": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.GuardDuty" }, "java": { "package": "software.amazon.awscdk.services.guardduty" }, "python": { "module": "aws_cdk.aws_guardduty" } } }, "aws-cdk-lib.aws_healthimaging": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.HealthImaging" }, "java": { "package": "software.amazon.awscdk.services.healthimaging" }, "python": { "module": "aws_cdk.aws_healthimaging" } } }, "aws-cdk-lib.aws_healthlake": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.HealthLake" }, "java": { "package": "software.amazon.awscdk.services.healthlake" }, "python": { "module": "aws_cdk.aws_healthlake" } } }, "aws-cdk-lib.aws_iam": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IAM" }, "java": { "package": "software.amazon.awscdk.services.iam" }, "python": { "module": "aws_cdk.aws_iam" } } }, "aws-cdk-lib.aws_identitystore": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IdentityStore" }, "java": { "package": "software.amazon.awscdk.services.identitystore" }, "python": { "module": "aws_cdk.aws_identitystore" } } }, "aws-cdk-lib.aws_imagebuilder": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ImageBuilder" }, "java": { "package": "software.amazon.awscdk.services.imagebuilder" }, "python": { "module": "aws_cdk.aws_imagebuilder" } } }, "aws-cdk-lib.aws_inspector": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Inspector" }, "java": { "package": "software.amazon.awscdk.services.inspector" }, "python": { "module": "aws_cdk.aws_inspector" } } }, "aws-cdk-lib.aws_inspectorv2": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.InspectorV2" }, "java": { "package": "software.amazon.awscdk.services.inspectorv2" }, "python": { "module": "aws_cdk.aws_inspectorv2" } } }, "aws-cdk-lib.aws_internetmonitor": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.InternetMonitor" }, "java": { "package": "software.amazon.awscdk.services.internetmonitor" }, "python": { "module": "aws_cdk.aws_internetmonitor" } } }, "aws-cdk-lib.aws_iot": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoT" }, "java": { "package": "software.amazon.awscdk.services.iot" }, "python": { "module": "aws_cdk.aws_iot" } } }, "aws-cdk-lib.aws_iot1click": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoT1Click" }, "java": { "package": "software.amazon.awscdk.services.iot1click" }, "python": { "module": "aws_cdk.aws_iot1click" } } }, "aws-cdk-lib.aws_iotanalytics": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoTAnalytics" }, "java": { "package": "software.amazon.awscdk.services.iotanalytics" }, "python": { "module": "aws_cdk.aws_iotanalytics" } } }, "aws-cdk-lib.aws_iotcoredeviceadvisor": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoTCoreDeviceAdvisor" }, "java": { "package": "software.amazon.awscdk.services.iotcoredeviceadvisor" }, "python": { "module": "aws_cdk.aws_iotcoredeviceadvisor" } } }, "aws-cdk-lib.aws_iotevents": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoTEvents" }, "java": { "package": "software.amazon.awscdk.services.iotevents" }, "python": { "module": "aws_cdk.aws_iotevents" } } }, "aws-cdk-lib.aws_iotfleethub": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoTFleetHub" }, "java": { "package": "software.amazon.awscdk.services.iotfleethub" }, "python": { "module": "aws_cdk.aws_iotfleethub" } } }, "aws-cdk-lib.aws_iotfleetwise": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoTFleetWise" }, "java": { "package": "software.amazon.awscdk.services.iotfleetwise" }, "python": { "module": "aws_cdk.aws_iotfleetwise" } } }, "aws-cdk-lib.aws_iotsitewise": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoTSiteWise" }, "java": { "package": "software.amazon.awscdk.services.iotsitewise" }, "python": { "module": "aws_cdk.aws_iotsitewise" } } }, "aws-cdk-lib.aws_iotthingsgraph": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoTThingsGraph" }, "java": { "package": "software.amazon.awscdk.services.iotthingsgraph" }, "python": { "module": "aws_cdk.aws_iotthingsgraph" } } }, "aws-cdk-lib.aws_iottwinmaker": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoTTwinMaker" }, "java": { "package": "software.amazon.awscdk.services.iottwinmaker" }, "python": { "module": "aws_cdk.aws_iottwinmaker" } } }, "aws-cdk-lib.aws_iotwireless": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IoTWireless" }, "java": { "package": "software.amazon.awscdk.services.iotwireless" }, "python": { "module": "aws_cdk.aws_iotwireless" } } }, "aws-cdk-lib.aws_ivs": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Ivs" }, "java": { "package": "software.amazon.awscdk.services.ivs" }, "python": { "module": "aws_cdk.aws_ivs" } } }, "aws-cdk-lib.aws_ivschat": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.IVSChat" }, "java": { "package": "software.amazon.awscdk.services.ivschat" }, "python": { "module": "aws_cdk.aws_ivschat" } } }, "aws-cdk-lib.aws_kafkaconnect": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.KafkaConnect" }, "java": { "package": "software.amazon.awscdk.services.kafkaconnect" }, "python": { "module": "aws_cdk.aws_kafkaconnect" } } }, "aws-cdk-lib.aws_kendra": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Kendra" }, "java": { "package": "software.amazon.awscdk.services.kendra" }, "python": { "module": "aws_cdk.aws_kendra" } } }, "aws-cdk-lib.aws_kendraranking": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.KendraRanking" }, "java": { "package": "software.amazon.awscdk.services.kendraranking" }, "python": { "module": "aws_cdk.aws_kendraranking" } } }, "aws-cdk-lib.aws_kinesis": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Kinesis" }, "java": { "package": "software.amazon.awscdk.services.kinesis" }, "python": { "module": "aws_cdk.aws_kinesis" } } }, "aws-cdk-lib.aws_kinesisanalytics": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.KinesisAnalytics" }, "java": { "package": "software.amazon.awscdk.services.kinesisanalytics" }, "python": { "module": "aws_cdk.aws_kinesisanalytics" } } }, "aws-cdk-lib.aws_kinesisanalyticsv2": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.KinesisAnalyticsV2" }, "java": { "package": "software.amazon.awscdk.services.kinesisanalyticsv2" }, "python": { "module": "aws_cdk.aws_kinesisanalyticsv2" } } }, "aws-cdk-lib.aws_kinesisfirehose": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.KinesisFirehose" }, "java": { "package": "software.amazon.awscdk.services.kinesisfirehose" }, "python": { "module": "aws_cdk.aws_kinesisfirehose" } } }, "aws-cdk-lib.aws_kinesisvideo": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.KinesisVideo" }, "java": { "package": "software.amazon.awscdk.services.kinesisvideo" }, "python": { "module": "aws_cdk.aws_kinesisvideo" } } }, "aws-cdk-lib.aws_kms": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.KMS" }, "java": { "package": "software.amazon.awscdk.services.kms" }, "python": { "module": "aws_cdk.aws_kms" } } }, "aws-cdk-lib.aws_lakeformation": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.LakeFormation" }, "java": { "package": "software.amazon.awscdk.services.lakeformation" }, "python": { "module": "aws_cdk.aws_lakeformation" } } }, "aws-cdk-lib.aws_lambda": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Lambda" }, "java": { "package": "software.amazon.awscdk.services.lambda" }, "python": { "module": "aws_cdk.aws_lambda" } } }, "aws-cdk-lib.aws_lambda_destinations": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Lambda.Destinations" }, "java": { "package": "software.amazon.awscdk.services.lambda.destinations" }, "python": { "module": "aws_cdk.aws_lambda_destinations" } } }, "aws-cdk-lib.aws_lambda_event_sources": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Lambda.EventSources" }, "java": { "package": "software.amazon.awscdk.services.lambda.eventsources" }, "python": { "module": "aws_cdk.aws_lambda_event_sources" } } }, "aws-cdk-lib.aws_lambda_nodejs": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Lambda.Nodejs" }, "java": { "package": "software.amazon.awscdk.services.lambda.nodejs" }, "python": { "module": "aws_cdk.aws_lambda_nodejs" } } }, "aws-cdk-lib.aws_lex": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Lex" }, "java": { "package": "software.amazon.awscdk.services.lex" }, "python": { "module": "aws_cdk.aws_lex" } } }, "aws-cdk-lib.aws_licensemanager": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.LicenseManager" }, "java": { "package": "software.amazon.awscdk.services.licensemanager" }, "python": { "module": "aws_cdk.aws_licensemanager" } } }, "aws-cdk-lib.aws_lightsail": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Lightsail" }, "java": { "package": "software.amazon.awscdk.services.lightsail" }, "python": { "module": "aws_cdk.aws_lightsail" } } }, "aws-cdk-lib.aws_location": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Location" }, "java": { "package": "software.amazon.awscdk.services.location" }, "python": { "module": "aws_cdk.aws_location" } } }, "aws-cdk-lib.aws_logs": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Logs" }, "java": { "package": "software.amazon.awscdk.services.logs" }, "python": { "module": "aws_cdk.aws_logs" } } }, "aws-cdk-lib.aws_logs_destinations": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Logs.Destinations" }, "java": { "package": "software.amazon.awscdk.services.logs.destinations" }, "python": { "module": "aws_cdk.aws_logs_destinations" } } }, "aws-cdk-lib.aws_lookoutequipment": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.LookoutEquipment" }, "java": { "package": "software.amazon.awscdk.services.lookoutequipment" }, "python": { "module": "aws_cdk.aws_lookoutequipment" } } }, "aws-cdk-lib.aws_lookoutmetrics": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.LookoutMetrics" }, "java": { "package": "software.amazon.awscdk.services.lookoutmetrics" }, "python": { "module": "aws_cdk.aws_lookoutmetrics" } } }, "aws-cdk-lib.aws_lookoutvision": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.LookoutVision" }, "java": { "package": "software.amazon.awscdk.services.lookoutvision" }, "python": { "module": "aws_cdk.aws_lookoutvision" } } }, "aws-cdk-lib.aws_m2": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.M2" }, "java": { "package": "software.amazon.awscdk.services.m2" }, "python": { "module": "aws_cdk.aws_m2" } } }, "aws-cdk-lib.aws_macie": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Macie" }, "java": { "package": "software.amazon.awscdk.services.macie" }, "python": { "module": "aws_cdk.aws_macie" } } }, "aws-cdk-lib.aws_managedblockchain": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ManagedBlockchain" }, "java": { "package": "software.amazon.awscdk.services.managedblockchain" }, "python": { "module": "aws_cdk.aws_managedblockchain" } } }, "aws-cdk-lib.aws_mediaconnect": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.MediaConnect" }, "java": { "package": "software.amazon.awscdk.services.mediaconnect" }, "python": { "module": "aws_cdk.aws_mediaconnect" } } }, "aws-cdk-lib.aws_mediaconvert": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.MediaConvert" }, "java": { "package": "software.amazon.awscdk.services.mediaconvert" }, "python": { "module": "aws_cdk.aws_mediaconvert" } } }, "aws-cdk-lib.aws_medialive": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.MediaLive" }, "java": { "package": "software.amazon.awscdk.services.medialive" }, "python": { "module": "aws_cdk.aws_medialive" } } }, "aws-cdk-lib.aws_mediapackage": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.MediaPackage" }, "java": { "package": "software.amazon.awscdk.services.mediapackage" }, "python": { "module": "aws_cdk.aws_mediapackage" } } }, "aws-cdk-lib.aws_mediapackagev2": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.MediaPackageV2" }, "java": { "package": "software.amazon.awscdk.services.mediapackagev2" }, "python": { "module": "aws_cdk.aws_mediapackagev2" } } }, "aws-cdk-lib.aws_mediastore": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.MediaStore" }, "java": { "package": "software.amazon.awscdk.services.mediastore" }, "python": { "module": "aws_cdk.aws_mediastore" } } }, "aws-cdk-lib.aws_mediatailor": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.MediaTailor" }, "java": { "package": "software.amazon.awscdk.services.mediatailor" }, "python": { "module": "aws_cdk.aws_mediatailor" } } }, "aws-cdk-lib.aws_memorydb": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.MemoryDB" }, "java": { "package": "software.amazon.awscdk.services.memorydb" }, "python": { "module": "aws_cdk.aws_memorydb" } } }, "aws-cdk-lib.aws_msk": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.MSK" }, "java": { "package": "software.amazon.awscdk.services.msk" }, "python": { "module": "aws_cdk.aws_msk" } } }, "aws-cdk-lib.aws_mwaa": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.MWAA" }, "java": { "package": "software.amazon.awscdk.services.mwaa" }, "python": { "module": "aws_cdk.aws_mwaa" } } }, "aws-cdk-lib.aws_neptune": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Neptune" }, "java": { "package": "software.amazon.awscdk.services.neptune" }, "python": { "module": "aws_cdk.aws_neptune" } } }, "aws-cdk-lib.aws_networkfirewall": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.NetworkFirewall" }, "java": { "package": "software.amazon.awscdk.services.networkfirewall" }, "python": { "module": "aws_cdk.aws_networkfirewall" } } }, "aws-cdk-lib.aws_networkmanager": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.NetworkManager" }, "java": { "package": "software.amazon.awscdk.services.networkmanager" }, "python": { "module": "aws_cdk.aws_networkmanager" } } }, "aws-cdk-lib.aws_nimblestudio": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.NimbleStudio" }, "java": { "package": "software.amazon.awscdk.services.nimblestudio" }, "python": { "module": "aws_cdk.aws_nimblestudio" } } }, "aws-cdk-lib.aws_oam": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Oam" }, "java": { "package": "software.amazon.awscdk.services.oam" }, "python": { "module": "aws_cdk.aws_oam" } } }, "aws-cdk-lib.aws_omics": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Omics" }, "java": { "package": "software.amazon.awscdk.services.omics" }, "python": { "module": "aws_cdk.aws_omics" } } }, "aws-cdk-lib.aws_opensearchserverless": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.OpenSearchServerless" }, "java": { "package": "software.amazon.awscdk.services.opensearchserverless" }, "python": { "module": "aws_cdk.aws_opensearchserverless" } } }, "aws-cdk-lib.aws_opensearchservice": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.OpenSearchService" }, "java": { "package": "software.amazon.awscdk.services.opensearchservice" }, "python": { "module": "aws_cdk.aws_opensearchservice" } } }, "aws-cdk-lib.aws_opsworks": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.OpsWorks" }, "java": { "package": "software.amazon.awscdk.services.opsworks" }, "python": { "module": "aws_cdk.aws_opsworks" } } }, "aws-cdk-lib.aws_opsworkscm": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.OpsWorksCM" }, "java": { "package": "software.amazon.awscdk.services.opsworkscm" }, "python": { "module": "aws_cdk.aws_opsworkscm" } } }, "aws-cdk-lib.aws_organizations": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Organizations" }, "java": { "package": "software.amazon.awscdk.services.organizations" }, "python": { "module": "aws_cdk.aws_organizations" } } }, "aws-cdk-lib.aws_osis": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.OSIS" }, "java": { "package": "software.amazon.awscdk.services.osis" }, "python": { "module": "aws_cdk.aws_osis" } } }, "aws-cdk-lib.aws_panorama": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Panorama" }, "java": { "package": "software.amazon.awscdk.services.panorama" }, "python": { "module": "aws_cdk.aws_panorama" } } }, "aws-cdk-lib.aws_pcaconnectorad": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.PCAConnectorAD" }, "java": { "package": "software.amazon.awscdk.services.pcaconnectorad" }, "python": { "module": "aws_cdk.aws_pcaconnectorad" } } }, "aws-cdk-lib.aws_personalize": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Personalize" }, "java": { "package": "software.amazon.awscdk.services.personalize" }, "python": { "module": "aws_cdk.aws_personalize" } } }, "aws-cdk-lib.aws_pinpoint": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Pinpoint" }, "java": { "package": "software.amazon.awscdk.services.pinpoint" }, "python": { "module": "aws_cdk.aws_pinpoint" } } }, "aws-cdk-lib.aws_pinpointemail": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.PinpointEmail" }, "java": { "package": "software.amazon.awscdk.services.pinpointemail" }, "python": { "module": "aws_cdk.aws_pinpointemail" } } }, "aws-cdk-lib.aws_pipes": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Pipes" }, "java": { "package": "software.amazon.awscdk.services.pipes" }, "python": { "module": "aws_cdk.aws_pipes" } } }, "aws-cdk-lib.aws_proton": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.Proton" }, "java": { "package": "software.amazon.awscdk.services.proton" }, "python": { "module": "aws_cdk.aws_proton" } } }, "aws-cdk-lib.aws_qldb": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.QLDB" }, "java": { "package": "software.amazon.awscdk.services.qldb" }, "python": { "module": "aws_cdk.aws_qldb" } } }, "aws-cdk-lib.aws_quicksight": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.QuickSight" }, "java": { "package": "software.amazon.awscdk.services.quicksight" }, "python": { "module": "aws_cdk.aws_quicksight" } } }, "aws-cdk-lib.aws_ram": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.RAM" }, "java": { "package": "software.amazon.awscdk.services.ram" }, "python": { "module": "aws_cdk.aws_ram" } } }, "aws-cdk-lib.aws_rds": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.RDS" }, "java": { "package": "software.amazon.awscdk.services.rds" }, "python": { "module": "aws_cdk.aws_rds" } } }, "aws-cdk-lib.aws_redshift": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Redshift" }, "java": { "package": "software.amazon.awscdk.services.redshift" }, "python": { "module": "aws_cdk.aws_redshift" } } }, "aws-cdk-lib.aws_redshiftserverless": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.RedshiftServerless" }, "java": { "package": "software.amazon.awscdk.services.redshiftserverless" }, "python": { "module": "aws_cdk.aws_redshiftserverless" } } }, "aws-cdk-lib.aws_refactorspaces": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.RefactorSpaces" }, "java": { "package": "software.amazon.awscdk.services.refactorspaces" }, "python": { "module": "aws_cdk.aws_refactorspaces" } } }, "aws-cdk-lib.aws_rekognition": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Rekognition" }, "java": { "package": "software.amazon.awscdk.services.rekognition" }, "python": { "module": "aws_cdk.aws_rekognition" } } }, "aws-cdk-lib.aws_resiliencehub": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ResilienceHub" }, "java": { "package": "software.amazon.awscdk.services.resiliencehub" }, "python": { "module": "aws_cdk.aws_resiliencehub" } } }, "aws-cdk-lib.aws_resourceexplorer2": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ResourceExplorer2" }, "java": { "package": "software.amazon.awscdk.services.resourceexplorer2" }, "python": { "module": "aws_cdk.aws_resourceexplorer2" } } }, "aws-cdk-lib.aws_resourcegroups": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ResourceGroups" }, "java": { "package": "software.amazon.awscdk.services.resourcegroups" }, "python": { "module": "aws_cdk.aws_resourcegroups" } } }, "aws-cdk-lib.aws_robomaker": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.RoboMaker" }, "java": { "package": "software.amazon.awscdk.services.robomaker" }, "python": { "module": "aws_cdk.aws_robomaker" } } }, "aws-cdk-lib.aws_rolesanywhere": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.RolesAnywhere" }, "java": { "package": "software.amazon.awscdk.services.rolesanywhere" }, "python": { "module": "aws_cdk.aws_rolesanywhere" } } }, "aws-cdk-lib.aws_route53": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Route53" }, "java": { "package": "software.amazon.awscdk.services.route53" }, "python": { "module": "aws_cdk.aws_route53" } } }, "aws-cdk-lib.aws_route53_patterns": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Route53.Patterns" }, "java": { "package": "software.amazon.awscdk.services.route53.patterns" }, "python": { "module": "aws_cdk.aws_route53_patterns" } } }, "aws-cdk-lib.aws_route53_targets": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Route53.Targets" }, "java": { "package": "software.amazon.awscdk.services.route53.targets" }, "python": { "module": "aws_cdk.aws_route53_targets" } } }, "aws-cdk-lib.aws_route53recoverycontrol": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Route53RecoveryControl" }, "java": { "package": "software.amazon.awscdk.services.route53recoverycontrol" }, "python": { "module": "aws_cdk.aws_route53recoverycontrol" } } }, "aws-cdk-lib.aws_route53recoveryreadiness": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Route53RecoveryReadiness" }, "java": { "package": "software.amazon.awscdk.services.route53recoveryreadiness" }, "python": { "module": "aws_cdk.aws_route53recoveryreadiness" } } }, "aws-cdk-lib.aws_route53resolver": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Route53Resolver" }, "java": { "package": "software.amazon.awscdk.services.route53resolver" }, "python": { "module": "aws_cdk.aws_route53resolver" } } }, "aws-cdk-lib.aws_rum": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.RUM" }, "java": { "package": "software.amazon.awscdk.services.rum" }, "python": { "module": "aws_cdk.aws_rum" } } }, "aws-cdk-lib.aws_s3": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.S3" }, "java": { "package": "software.amazon.awscdk.services.s3" }, "python": { "module": "aws_cdk.aws_s3" } } }, "aws-cdk-lib.aws_s3_assets": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.S3.Assets" }, "java": { "package": "software.amazon.awscdk.services.s3.assets" }, "python": { "module": "aws_cdk.aws_s3_assets" } } }, "aws-cdk-lib.aws_s3_deployment": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.S3.Deployment" }, "java": { "package": "software.amazon.awscdk.services.s3.deployment" }, "python": { "module": "aws_cdk.aws_s3_deployment" } } }, "aws-cdk-lib.aws_s3_notifications": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.S3.Notifications" }, "java": { "package": "software.amazon.awscdk.services.s3.notifications" }, "python": { "module": "aws_cdk.aws_s3_notifications" } } }, "aws-cdk-lib.aws_s3express": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.S3Express" }, "java": { "package": "software.amazon.awscdk.services.s3express" }, "python": { "module": "aws_cdk.aws_s3express" } } }, "aws-cdk-lib.aws_s3objectlambda": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.S3ObjectLambda" }, "java": { "package": "software.amazon.awscdk.services.s3objectlambda" }, "python": { "module": "aws_cdk.aws_s3objectlambda" } } }, "aws-cdk-lib.aws_s3outposts": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.S3Outposts" }, "java": { "package": "software.amazon.awscdk.services.s3outposts" }, "python": { "module": "aws_cdk.aws_s3outposts" } } }, "aws-cdk-lib.aws_sagemaker": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Sagemaker" }, "java": { "package": "software.amazon.awscdk.services.sagemaker" }, "python": { "module": "aws_cdk.aws_sagemaker" } } }, "aws-cdk-lib.aws_sam": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SAM" }, "java": { "package": "software.amazon.awscdk.services.sam" }, "python": { "module": "aws_cdk.aws_sam" } } }, "aws-cdk-lib.aws_scheduler": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Scheduler" }, "java": { "package": "software.amazon.awscdk.services.scheduler" }, "python": { "module": "aws_cdk.aws_scheduler" } } }, "aws-cdk-lib.aws_sdb": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SDB" }, "java": { "package": "software.amazon.awscdk.services.sdb" }, "python": { "module": "aws_cdk.aws_sdb" } } }, "aws-cdk-lib.aws_secretsmanager": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SecretsManager" }, "java": { "package": "software.amazon.awscdk.services.secretsmanager" }, "python": { "module": "aws_cdk.aws_secretsmanager" } } }, "aws-cdk-lib.aws_securityhub": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SecurityHub" }, "java": { "package": "software.amazon.awscdk.services.securityhub" }, "python": { "module": "aws_cdk.aws_securityhub" } } }, "aws-cdk-lib.aws_servicecatalog": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Servicecatalog" }, "java": { "package": "software.amazon.awscdk.services.servicecatalog" }, "python": { "module": "aws_cdk.aws_servicecatalog" } } }, "aws-cdk-lib.aws_servicecatalogappregistry": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Servicecatalogappregistry" }, "java": { "package": "software.amazon.awscdk.services.servicecatalogappregistry" }, "python": { "module": "aws_cdk.aws_servicecatalogappregistry" } } }, "aws-cdk-lib.aws_servicediscovery": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.ServiceDiscovery" }, "java": { "package": "software.amazon.awscdk.services.servicediscovery" }, "python": { "module": "aws_cdk.aws_servicediscovery" } } }, "aws-cdk-lib.aws_ses": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SES" }, "java": { "package": "software.amazon.awscdk.services.ses" }, "python": { "module": "aws_cdk.aws_ses" } } }, "aws-cdk-lib.aws_ses_actions": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SES.Actions" }, "java": { "package": "software.amazon.awscdk.services.ses.actions" }, "python": { "module": "aws_cdk.aws_ses_actions" } } }, "aws-cdk-lib.aws_shield": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.Shield" }, "java": { "package": "software.amazon.awscdk.services.shield" }, "python": { "module": "aws_cdk.aws_shield" } } }, "aws-cdk-lib.aws_signer": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Signer" }, "java": { "package": "software.amazon.awscdk.services.signer" }, "python": { "module": "aws_cdk.aws_signer" } } }, "aws-cdk-lib.aws_simspaceweaver": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SimSpaceWeaver" }, "java": { "package": "software.amazon.awscdk.services.simspaceweaver" }, "python": { "module": "aws_cdk.aws_simspaceweaver" } } }, "aws-cdk-lib.aws_sns": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SNS" }, "java": { "package": "software.amazon.awscdk.services.sns" }, "python": { "module": "aws_cdk.aws_sns" } } }, "aws-cdk-lib.aws_sns_subscriptions": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SNS.Subscriptions" }, "java": { "package": "software.amazon.awscdk.services.sns.subscriptions" }, "python": { "module": "aws_cdk.aws_sns_subscriptions" } } }, "aws-cdk-lib.aws_sqs": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SQS" }, "java": { "package": "software.amazon.awscdk.services.sqs" }, "python": { "module": "aws_cdk.aws_sqs" } } }, "aws-cdk-lib.aws_ssm": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SSM" }, "java": { "package": "software.amazon.awscdk.services.ssm" }, "python": { "module": "aws_cdk.aws_ssm" } } }, "aws-cdk-lib.aws_ssmcontacts": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SSMContacts" }, "java": { "package": "software.amazon.awscdk.services.ssmcontacts" }, "python": { "module": "aws_cdk.aws_ssmcontacts" } } }, "aws-cdk-lib.aws_ssmincidents": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SSMIncidents" }, "java": { "package": "software.amazon.awscdk.services.ssmincidents" }, "python": { "module": "aws_cdk.aws_ssmincidents" } } }, "aws-cdk-lib.aws_sso": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SSO" }, "java": { "package": "software.amazon.awscdk.services.sso" }, "python": { "module": "aws_cdk.aws_sso" } } }, "aws-cdk-lib.aws_stepfunctions": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.StepFunctions" }, "java": { "package": "software.amazon.awscdk.services.stepfunctions" }, "python": { "module": "aws_cdk.aws_stepfunctions" } } }, "aws-cdk-lib.aws_stepfunctions_tasks": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.StepFunctions.Tasks" }, "java": { "package": "software.amazon.awscdk.services.stepfunctions.tasks" }, "python": { "module": "aws_cdk.aws_stepfunctions_tasks" } } }, "aws-cdk-lib.aws_supportapp": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SupportApp" }, "java": { "package": "software.amazon.awscdk.services.supportapp" }, "python": { "module": "aws_cdk.aws_supportapp" } } }, "aws-cdk-lib.aws_synthetics": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Synthetics" }, "java": { "package": "software.amazon.awscdk.services.synthetics" }, "python": { "module": "aws_cdk.aws_synthetics" } } }, "aws-cdk-lib.aws_systemsmanagersap": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.SystemsManagerSAP" }, "java": { "package": "software.amazon.awscdk.services.systemsmanagersap" }, "python": { "module": "aws_cdk.aws_systemsmanagersap" } } }, "aws-cdk-lib.aws_timestream": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Timestream" }, "java": { "package": "software.amazon.awscdk.services.timestream" }, "python": { "module": "aws_cdk.aws_timestream" } } }, "aws-cdk-lib.aws_transfer": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Transfer" }, "java": { "package": "software.amazon.awscdk.services.transfer" }, "python": { "module": "aws_cdk.aws_transfer" } } }, "aws-cdk-lib.aws_verifiedpermissions": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.VerifiedPermissions" }, "java": { "package": "software.amazon.awscdk.services.verifiedpermissions" }, "python": { "module": "aws_cdk.aws_verifiedpermissions" } } }, "aws-cdk-lib.aws_voiceid": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.VoiceID" }, "java": { "package": "software.amazon.awscdk.services.voiceid" }, "python": { "module": "aws_cdk.aws_voiceid" } } }, "aws-cdk-lib.aws_vpclattice": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.VpcLattice" }, "java": { "package": "software.amazon.awscdk.services.vpclattice" }, "python": { "module": "aws_cdk.aws_vpclattice" } } }, "aws-cdk-lib.aws_waf": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.WAF" }, "java": { "package": "software.amazon.awscdk.services.waf" }, "python": { "module": "aws_cdk.aws_waf" } } }, "aws-cdk-lib.aws_wafregional": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.WAFRegional" }, "java": { "package": "software.amazon.awscdk.services.waf.regional" }, "python": { "module": "aws_cdk.aws_wafregional" } } }, "aws-cdk-lib.aws_wafv2": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.WAFv2" }, "java": { "package": "software.amazon.awscdk.services.wafv2" }, "python": { "module": "aws_cdk.aws_wafv2" } } }, "aws-cdk-lib.aws_wisdom": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.Wisdom" }, "java": { "package": "software.amazon.awscdk.services.wisdom" }, "python": { "module": "aws_cdk.aws_wisdom" } } }, "aws-cdk-lib.aws_workspaces": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.WorkSpaces" }, "java": { "package": "software.amazon.awscdk.services.workspaces" }, "python": { "module": "aws_cdk.aws_workspaces" } } }, "aws-cdk-lib.aws_workspacesthinclient": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.WorkSpacesThinClient" }, "java": { "package": "software.amazon.awscdk.services.workspacesthinclient" }, "python": { "module": "aws_cdk.aws_workspacesthinclient" } } }, "aws-cdk-lib.aws_workspacesweb": { "targets": { "dotnet": { "package": "Amazon.CDK.AWS.WorkSpacesWeb" }, "java": { "package": "software.amazon.awscdk.services.workspacesweb" }, "python": { "module": "aws_cdk.aws_workspacesweb" } } }, "aws-cdk-lib.aws_xray": { "targets": { "dotnet": { "namespace": "Amazon.CDK.AWS.XRay" }, "java": { "package": "software.amazon.awscdk.services.xray" }, "python": { "module": "aws_cdk.aws_xray" } } }, "aws-cdk-lib.cloud_assembly_schema": { "targets": { "dotnet": { "namespace": "Amazon.CDK.CloudAssembly.Schema" }, "java": { "package": "software.amazon.awscdk.cloudassembly.schema" }, "python": { "module": "aws_cdk.cloud_assembly_schema" } } }, "aws-cdk-lib.cloudformation_include": { "targets": { "dotnet": { "namespace": "Amazon.CDK.CloudFormation.Include" }, "java": { "package": "software.amazon.awscdk.cloudformation.include" }, "python": { "module": "aws_cdk.cloudformation_include" } } }, "aws-cdk-lib.custom_resources": { "targets": { "dotnet": { "namespace": "Amazon.CDK.CustomResources" }, "java": { "package": "software.amazon.awscdk.customresources" }, "python": { "module": "aws_cdk.custom_resources" } } }, "aws-cdk-lib.cx_api": { "targets": { "dotnet": { "namespace": "Amazon.CDK.CXAPI" }, "java": { "package": "software.amazon.awscdk.cxapi" }, "python": { "module": "aws_cdk.cx_api" } } }, "aws-cdk-lib.lambda_layer_awscli": { "targets": { "dotnet": { "namespace": "Amazon.CDK.LambdaLayer.AwsCli" }, "java": { "package": "software.amazon.awscdk.lambdalayer.awscli" }, "python": { "module": "aws_cdk.lambda_layer_awscli" } } }, "aws-cdk-lib.lambda_layer_kubectl": { "targets": { "dotnet": { "namespace": "Amazon.CDK.LambdaLayer.Kubectl" }, "java": { "package": "software.amazon.awscdk.lambdalayer.kubectl" }, "python": { "module": "aws_cdk.lambda_layer_kubectl" } } }, "aws-cdk-lib.lambda_layer_node_proxy_agent": { "targets": { "dotnet": { "namespace": "Amazon.CDK.LambdaLayer.NodeProxyAgent" }, "java": { "package": "software.amazon.awscdk.lambda.layer.node.proxy.agent" }, "python": { "module": "aws_cdk.lambda_layer_node_proxy_agent" } } }, "aws-cdk-lib.pipelines": { "targets": { "dotnet": { "namespace": "Amazon.CDK.Pipelines" }, "java": { "package": "software.amazon.awscdk.pipelines" }, "python": { "module": "aws_cdk.pipelines" } } }, "aws-cdk-lib.region_info": { "targets": { "dotnet": { "namespace": "Amazon.CDK.RegionInfo" }, "java": { "package": "software.amazon.awscdk.regioninfo" }, "python": { "module": "aws_cdk.region_info" } } }, "aws-cdk-lib.triggers": { "targets": { "dotnet": { "namespace": "Amazon.CDK.Triggers" }, "java": { "package": "software.amazon.awscdk.triggers" }, "python": { "module": "aws_cdk.triggers" } } } }, "targets": { "dotnet": { "iconUrl": "https://raw.githubusercontent.com/aws/aws-cdk/main/logo/default-256-dark.png", "namespace": "Amazon.CDK", "packageId": "Amazon.CDK.Lib" }, "go": { "moduleName": "github.com/aws/aws-cdk-go", "packageName": "awscdk" }, "java": { "maven": { "artifactId": "aws-cdk-lib", "groupId": "software.amazon.awscdk" }, "package": "software.amazon.awscdk" }, "js": { "npm": "aws-cdk-lib" }, "python": { "distName": "aws-cdk-lib", "module": "aws_cdk" } } }, "constructs": { "targets": { "dotnet": { "namespace": "Constructs", "packageId": "Constructs" }, "go": { "moduleName": "github.com/aws/constructs-go" }, "java": { "maven": { "artifactId": "constructs", "groupId": "software.constructs" }, "package": "software.constructs" }, "js": { "npm": "constructs" }, "python": { "distName": "constructs", "module": "constructs" } } } }, "description": "L3 CDK Constructs used to build data solutions with AWS", "docs": { "stability": "stable" }, "homepage": "https://awslabs.github.io/data-solutions-framework-on-aws/", "jsiiVersion": "5.3.7 (build e512679)", "keywords": [ "analytics", "aws", "awscdk", "cdk", "constructs", "data", "datalake" ], "license": "Apache-2.0", "metadata": { "jsii": { "pacmak": { "hasDefaultInterfaces": true } }, "tscRootDir": "src" }, "name": "aws-dsf", "readme": { "markdown": "# Data solutions framework on AWS\n\n\nData solutions framework on AWS (DSF on AWS) is a framework for implementation and delivery of data solutions with built-in AWS best practices. DSF on AWS is an abstraction atop AWS services based on [AWS Cloud Development Kit](https://aws.amazon.com/cdk/) (CDK) L3 constructs, packaged as a library.\n\n➡️ **More information on our [website](https://awslabs.github.io/data-solutions-framework-on-aws)**\n" }, "repository": { "directory": "framework", "type": "git", "url": "https://github.com/awslabs/data-solutions-framework-on-aws.git" }, "schema": "jsii/0.10.0", "submodules": { "aws-dsf.consumption": { "locationInModule": { "filename": "src/index.ts", "line": 8 }, "readme": { "markdown": "[//]: # (consumption.redshift-serverless-namespace)\n# RedshiftServerlessNamespace\n\nA [Redshift Serverless Namespace](https://docs.aws.amazon.com/redshift/latest/mgmt/serverless-workgroup-namespace.html) with secrets manager integration for admin credentials management and rotation.\n\n## Overview\n\n`RedshiftServerlessNamespace` is a [Redshift Serverless Namespace](https://docs.aws.amazon.com/redshift/latest/mgmt/serverless-workgroup-namespace.html) with the following options:\n- Encrypt data with a customer managed KMS Key.\n- Create Redshift superuser credentials managed by Redshift service: stored in Secrets Manager, encrypted with a KMS Key, and with automatic rotation.\n- Attach multiple IAM roles that can be used by Redshift Serverless users to interact with other AWS services.\n- Set an [IAM role as default](https://docs.aws.amazon.com/redshift/latest/mgmt/default-iam-role.html)\n\n## Usage\n\n```ts lit=src/consumption/examples/redshift-serverless-namespace-default.lit.ts\nclass ExampleDefaultRedshiftServerlessNamespaceStack extends Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id)\n new dsf.consumption.RedshiftServerlessNamespace(this, \"RedshiftServerlessNamespace\", {\n dbName: \"database\",\n name: \"example-namespace\"\n })\n }\n}\n```\n\n## Attaching IAM Roles to Redshift Serverless Namespace\n\nTo allow Redshift Serverless to access other AWS services on your behalf (eg. data ingestion from S3 via the COPY command, accessing data in S3 via Redshift Spectrum, exporting data from Redshift to S3 via the UNLOAD command.), the preferred method is to specify an IAM role. High-level steps are as follows:\n\n1. Create an IAM role with a trust relationship of `redshift.amazonaws.com`.\n2. Attach policy/permissions to the role to give it access to specific AWS services.\n3. Configure the role when creating the Redshift Serverless Namespace\n4. Run the relevant SQL command referencing the attached IAM role via its ARN (or the `default` keyword if a default IAM role is configured)\n\n```ts lit=src/consumption/examples/redshift-serverless-namespace-roles.lit.ts\nclass ExampleRedshiftServerlessNamespaceRolesStack extends Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id)\n\n const bucket = new Bucket(this, \"ExampleBucket\")\n\n const ingestionRole = new Role(this, \"IngestionRole\", {\n assumedBy: new ServicePrincipal(\"redshift.amazonaws.com\"),\n managedPolicies: [\n ManagedPolicy.fromAwsManagedPolicyName(\"AmazonRedshiftAllCommandsFullAccess\")\n ]\n })\n\n bucket.grantRead(ingestionRole)\n\n new dsf.consumption.RedshiftServerlessNamespace(this, \"RedshiftServerlessNamespace\", {\n dbName: \"database\",\n name: \"example-namespace\",\n defaultIAMRole: ingestionRole\n })\n }\n}\n```\n\n[//]: # (consumption.redshift-serverless-workgroup)\n# RedshiftServerlessWorkgroup\n\nA [Redshift Serverless Workgroup](https://docs.aws.amazon.com/redshift/latest/mgmt/serverless-workgroup-namespace.html) with helpers method for Redshift administration.\n\n## Overview\n`RedshiftServerlessWorkgroup` is a [Redshift Serverless Workgroup](https://docs.aws.amazon.com/redshift/latest/mgmt/serverless-workgroup-namespace.html) with the following options/capabilities:\n- Deployed in a VPC in private subnets. The network configuation can be customized.\n- Provide helper methods for running SQL commands via the Redshift Data API. Commands can be custom or predefined for common administration tasks like creating and granting roles.\n- Initialize a Glue Data Catalog integration with auto crawling via Glue Crawlers. This would allow tables in Redshift Serverless to appear in the [Glue Data Catalog](https://docs.aws.amazon.com/glue/latest/dg/catalog-and-crawler.html) for the purposes of discovery and integration.\n\n## Usage\n\n```ts lit=src/consumption/examples/redshift-serverless-workgroup-default.lit.ts\nclass ExampleDefaultRedshiftServerlessWorkgroupStack extends Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id)\n\n const namespace = new dsf.consumption.RedshiftServerlessNamespace(this, 'DefaultRedshiftServerlessNamespace', {\n name: \"default\",\n dbName: 'defaultdb',\n })\n\n new dsf.consumption.RedshiftServerlessWorkgroup(this, \"DefaultRedshiftServerlessWorkgroup\", {\n name: \"default\",\n namespace: namespace,\n })\n }\n}\n```\n\n## Bootstrapping Redshift Serverless w/ RedshiftData Construct\n\nThe `RedshiftData` construct allows custom SQLs to run against the `RedshiftServerlessWorkgroup` via the Data API. This allows users to bootstrap Redshift directly from CDK.\n\nThe `RedshitData` construct provides the following helpers for bootstrapping Redshift databases:\n- Run a custom SQL command\n- Create Redshift roles\n- Grant Redshift roles full access to schemas\n- Grant Redshift roles read only access\n- Run a COPY command to load data\n\n```ts lit=src/consumption/examples/redshift-serverless-workgroup-bootstrap.lit.ts\n const workgroup = new dsf.consumption.RedshiftServerlessWorkgroup(this, \"DefaultRedshiftServerlessWorkgroup\", {\n name: \"default\",\n namespace: namespace,\n })\n\n // Initialize the Redshift Data API\n const dataAccess = workgroup.accessData('DataApi', true)\n\n // Run a custom SQL to create a customer table\n const createTable = dataAccess.runCustomSQL('CreateCustomerTable', \"defaultdb\",\n `\n CREATE TABLE customer(\n customer_id varchar(50),\n salutation varchar(5),\n first_name varchar(50),\n last_name varchar(50),\n email_address varchar(100)\n )\n diststyle even\n `,\n \"drop table customer\"\n );\n\n // Run a COPY command to load data into the customer table\n const ingestion = dataAccess.ingestData('ExampleCopy', \"defaultdb\", \"customer\", bucket, \"data-products/customer/\", \"csv ignoreheader 1\");\n\n // Add dependencies between Redshift Data API commands because CDK cannot infer them\n ingestion.node.addDependency(createTable);\n\n // Create an engineering role in the defaultdb\n const dbRole = dataAccess.createDbRole('EngineeringRole', 'defaultdb', 'engineering');\n\n // Grant the engineering role full access to the public schema in the defaultdb\n const dbSchema = dataAccess.grantDbSchemaToRole('EngineeringGrant', 'defaultdb', 'public', 'engineering');\n\n // Enforce dependencies\n dbSchema.node.addDependency(dbRole);\n```\n\n## Cataloging Redshift Serverless Tables\n\nRedshift tables and databases can also be automatically catalog in Glue Data Catalog using an helper method. This method creates a Glue Catalog database as well as a crawler to populate the database with table metadata from your Redshift database.\n\nThe default value of the path that the crawler would use is `/public/%` which translates to all the tables in the public schema. Please refer to the [crawler documentation](https://docs.aws.amazon.com/glue/latest/dg/define-crawler.html#define-crawler-choose-data-sources) for more information for JDBC data sources.\n\n```ts lit=src/consumption/examples/redshift-serverless-workgroup-catalog.lit.ts\nclass ExampleRedshiftServerlessWorkgroupCatalogStack extends Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id)\n\n const namespace = new dsf.consumption.RedshiftServerlessNamespace(this, 'DefaultRedshiftServerlessNamespace', {\n name: \"default\",\n dbName: 'defaultdb',\n })\n\n const workgroup = new dsf.consumption.RedshiftServerlessWorkgroup(this, \"DefaultRedshiftServerlessWorkgroup\", {\n name: \"default\",\n namespace: namespace,\n })\n\n workgroup.catalogTables('RedshiftCatalog', \"example-redshift-db\", 'defaultdb/public/%')\n }\n}\n```\n\n[//]: # (consumption.athena-workgroup)\n# Athena Workgroup\n\nAn [Amazon Athena workgroup](https://docs.aws.amazon.com/athena/latest/ug/manage-queries-control-costs-with-workgroups.html) with provided configuration.\n\n`AthenaWorkGroup` provides Athena workgroup configuration with best-practices:\n- Amazon S3 bucket for query results, based on [`AnalyticsBucket`](https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Storage/analytics-bucket).\n- Query results are encrypted using AWS KMS Key.\n- Execution Role for the PySpark query engine.\n- A grant method to allow principals to run queries.\n\n## Usage\n\n```ts lit=src/consumption/examples/athena-workgroup-default.lit.ts\nnew dsf.consumption.AthenaWorkGroup(this, 'AthenaWorkGroupDefault', {\n name: 'athena-default',\n resultLocationPrefix: 'athena-default-results/'\n})\n```\n\n## User provided S3 bucket for query results\n\nYou can provide your own S3 bucket for query results. If you do so, you are required to provide a KMS Key that will be used to encrypt query results.\n\n:::caution Results encryption\nIf you provide your own S3 bucket, you also need to provide KMS encryption key to encrypt query results. You also need to\ngrant access to this key for AthenaWorkGroup's executionRole (if Spark engine is used), or for principals that were granted to run\nqueries using AthenaWorkGroup's `grantRunQueries` method.\n:::caution\n\nYou can also decide to provide your KMS Key to encrypt query results with S3 bucket that is provided by the construct (i.e. if you are not providing your own S3 bucket).\n\n```ts lit=src/consumption/examples/athena-workgroup-user-bucket.lit.ts\nnew dsf.consumption.AthenaWorkGroup(this, 'AthenaWorkGroupDefault', {\n name: 'athena-user-bucket',\n resultBucket: userResultsBucket,\n resultsEncryptionKey: userDataKey,\n resultLocationPrefix: 'athena-wg-results/'\n})\n```\n\n## Apache Spark (PySpark) Engine version\n\nYou can choose Athena query engine from the available options:\n- [Athena engine version 3](https://docs.aws.amazon.com/athena/latest/ug/engine-versions-reference-0003.html)\n- [PySpark engine version 3](https://docs.aws.amazon.com/athena/latest/ug/notebooks-spark.html)\n\nThe `default` is set to `AUTO` which will choose Athena engine version 3.\n\nIf you wish to change query engine to PySpark, you will also be able to access the `executionRole` IAM Role that will be created for you if you don't provide it.\nYou can access the execution role via `executionRole` property.\n\n```ts lit=src/consumption/examples/athena-workgroup-spark.lit.ts\n const sparkEngineVersion = dsf.consumption.EngineVersion.PYSPARK_V3\n\n new dsf.consumption.AthenaWorkGroup(this, 'AthenaWorkGroupSpark', {\n name: 'athena-spark',\n engineVersion: sparkEngineVersion,\n resultLocationPrefix: 'athena-wg-results/'\n })\n```\n\n## Construct properties\n\nYou can leverage different properties to customize your Athena workgroup. For example, you can use `resultsRetentionPeriod` to specify the retention period for your query results. You can provide your KMS Key for encryption even if you use provided results bucket. You can explore other properties available in `AthenaWorkGroupProps`.\n\n```ts lit=src/consumption/examples/athena-workgroup-properties.lit.ts\nnew dsf.consumption.AthenaWorkGroup(this, 'AthenaWorkGroupProperties', {\n name: 'athena-properties',\n bytesScannedCutoffPerQuery: 104857600,\n resultLocationPrefix: 'athena-results/',\n resultsEncryptionKey: userDataKey,\n resultsRetentionPeriod: Duration.days(1),\n})\n```\n\n## Grant permission to run queries\n\nWe provide `grantRunQueries` method to grant permission to principals to run queries using the workgroup.\n\n```ts lit=src/consumption/examples/athena-workgroup-grant.lit.ts\n const athenaWg = new dsf.consumption.AthenaWorkGroup(this, 'AthenaWorkGroupGrant', {\n name: 'athena-grant',\n resultLocationPrefix: 'athena-results/',\n })\n\n athenaWg.grantRunQueries(athenaExampleRole)\n```\n\n\n## Workgroup removal\n\nYou can specify if Athena Workgroup construct resources should be deleted when CDK Stack is destroyed using `removalPolicy`. To have an additional layer of protection, we require users to set a global context value for data removal in their CDK applications.\n\nAthena workgroup will be destroyed only if **both** the removal policy parameter of the construct and DSF global removal policy are set to remove objects.\n\nIf set to be destroyed, Athena workgroup construct will use `recursiveDeleteOption`, that will delete the workgroup and its contents even if it contains any named queries.\n\nYou can set `@data-solutions-framework-on-aws/removeDataOnDestroy` (`true` or `false`) global data removal policy in `cdk.json`:\n\n```json title=\"cdk.json\"\n{\n \"context\": {\n \"@data-solutions-framework-on-aws/removeDataOnDestroy\": true\n }\n}\n```\n" }, "symbolId": "src/consumption/index:" }, "aws-dsf.governance": { "locationInModule": { "filename": "src/index.ts", "line": 5 }, "readme": { "markdown": "[//]: # (governance.data-catalog-database)\n# DataCatalogDatabase\n\nAWS Glue Catalog database for an Amazon S3 dataset.\n\n## Overview\n\n`DataCatalogDatabase` is an [AWS Glue Data Catalog Database](https://docs.aws.amazon.com/glue/latest/dg/define-database.html) configured for an Amazon S3 based dataset:\n- The database default location is pointing to an S3 bucket location `s3:////`\n- The database can store various tables structured in their respective prefixes, for example: `s3://///`\n- By default, a database level crawler is scheduled to run once a day (00:01h local timezone). The crawler can be disabled and the schedule/frequency of the crawler can be modified with a cron expression.\n\n![Data Catalog Database](../../../website/static/img/adsf-data-catalog.png)\n\n:::caution Data Catalog encryption\nThe AWS Glue Data Catalog resources created by the `DataCatalogDatabase` construct are not encrypted because the encryption is only available at the catalog level. Changing the encryption at the catalog level has a wide impact on existing Glue resources and producers/consumers. Similarly, changing the encryption configuration at the catalog level after this construct is deployed can break all the resources created as part of DSF on AWS.\n:::caution\n\n## Usage\n\n```ts lit=src/governance/examples/data-catalog-database-default.lit.ts\nclass ExampleDefaultDataCatalogDatabaseStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n const bucket = new Bucket(this, 'DataCatalogBucket');\n\n new dsf.governance.DataCatalogDatabase(this, 'DataCatalogDatabase', {\n locationBucket: bucket,\n locationPrefix: '/databasePath',\n name: 'example-db',\n });\n }\n}\n```\n\n## Modifying the crawler behavior\n\nYou can change the default configuration of the AWS Glue Crawler to match your requirements:\n* Enable or disable the crawler\n* Change the crawler run frequency\n* Provide your own key to encrypt the crawler logs\n\n```ts lit=src/governance/examples/data-catalog-database-crawler.lit.ts\n const encryptionKey = new Key(this, 'CrawlerLogEncryptionKey');\n\n new dsf.governance.DataCatalogDatabase(this, 'DataCatalogDatabase', {\n locationBucket: bucket,\n locationPrefix: '/databasePath',\n name: 'example-db',\n autoCrawl: true,\n autoCrawlSchedule: {\n scheduleExpression: 'cron(1 0 * * ? *)',\n },\n crawlerLogEncryptionKey: encryptionKey,\n crawlerTableLevelDepth: 3,\n });\n```\n\n[//]: # (governance.data-lake-catalog)\n# DataLakeCatalog\n\nAWS Glue Catalog databases on top of a DataLakeStorage.\n\n## Overview\n\n`DataLakeCatalog` is a data catalog for your data lake. It's a set of [AWS Glue Data Catalog Databases](https://docs.aws.amazon.com/glue/latest/dg/define-database.html) configured on top of a [`DataLakeStorage`](../storage/README.md#datalakestorage).\nThe construct creates three databases pointing to the respective medallion layers (bronze, silve or gold) of the `DataLakeStorage`:\n- The database default location is pointing to the corresponding S3 bucket location `s3:////`\n- By default, each database has an active crawler scheduled to run once a day (00:01h local timezone). The crawler can be disabled and the schedule/frequency of the crawler can be modified with a cron expression.\n\n![Data Lake Catalog](../../../website/static/img/adsf-data-lake-catalog.png)\n\n:::caution Data Catalog encryption\nThe AWS Glue Data Catalog resources created by the `DataCatalogDatabase` construct are not encrypted because the encryption is only available at the catalog level. Changing the encryption at the catalog level has a wide impact on existing Glue resources and producers/consumers. Similarly, changing the encryption configuration at the catalog level after this construct is deployed can break all the resources created as part of DSF on AWS.\n:::caution\n\n## Usage\n\n```ts lit=src/governance/examples/data-lake-catalog-default.lit.ts\nclass ExampleDefaultDataLakeCatalogStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n const storage = new dsf.storage.DataLakeStorage(this, 'MyDataLakeStorage');\n\n new dsf.governance.DataLakeCatalog(this, 'DataCatalog', {\n dataLakeStorage: storage,\n });\n }\n}\n```\n\n## Modifying the crawlers behavior for the entire catalog\n\nYou can change the default configuration of the AWS Glue Crawlers associated with the different databases to match your requirements:\n* Enable or disable the crawlers\n* Change the crawlers run frequency\n* Provide your own key to encrypt the crawlers logs\n\nThe parameters apply to the three databases, if you need fine-grained configuration per database, you can use the [DataCatalogDatabase](#datacatalogdatabase) construct.\n\n```ts lit=src/governance/examples/data-lake-catalog-crawler.lit.ts\n const encryptionKey = new Key(this, 'CrawlerLogEncryptionKey');\n\n new dsf.governance.DataLakeCatalog(this, 'DataCatalog', {\n dataLakeStorage: storage,\n autoCrawl: true,\n autoCrawlSchedule: {\n scheduleExpression: 'cron(1 0 * * ? *)',\n },\n crawlerLogEncryptionKey: encryptionKey,\n crawlerTableLevelDepth: 3,\n });\n```\n" }, "symbolId": "src/governance/index:" }, "aws-dsf.processing": { "locationInModule": { "filename": "src/index.ts", "line": 6 }, "readme": { "markdown": "[//]: # (processing.spark-emr-runtime-serverless)\n# Spark EMR Serverless Runtime\n\nA [Spark EMR Serverless Application](https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/getting-started.html) with IAM roles and permissions helpers.\n\n## Overview\n\nThe construct creates a Spark EMR Serverless Application, with the latest EMR runtime as the default runtime. You can change the runtime by passing your own as a `Resource property` to construct initializer. It also provides methods to create a principal or grant an existing principal (ie IAM Role or IAM User) with the permission to start a job on this EMR Serverless application.\n\nThe construct creates a default VPC that is used by EMR Serverless Application. The VPC has `10.0.0.0/16` CIDR range, and comes with an S3 VPC Endpoint Gateway attached to it. The construct also creates a security group for the EMR Serverless Application. You can override this by defining your own `NetworkConfiguration` as defined in the `Resource properties` of the construct initializer.\n\nThe construct has the following interfaces:\n\n* A construct Initializer that takes an object as `Resource properties` to modify the default properties. The properties are defined in `SparkEmrServerlessRuntimeProps` interface.\n* A method to create an execution role for EMR Serverless. The execution role is scoped down to the EMR Serverless Application ARN created by the construct.\n* A method that takes an IAM role to call the `StartJobRun`, and monitors the status of the job.\n * The IAM policies attached to the provided IAM role is as [follow](https://github.com/awslabs/data-solutions-framework-on-aws/blob/c965202f48088f5ae51ce0e719cf92adefac94ac/framework/src/processing/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts#L117).\n * The role has a `PassRole` permission scoped as [follow](https://github.com/awslabs/data-solutions-framework-on-aws/blob/c965202f48088f5ae51ce0e719cf92adefac94ac/framework/src/processing/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts#L106).\n\nThe construct has the following attributes:\n\n* applicationArn: EMR Serverless Application ARN\n* applicationId: EMR Serverless Application ID\n* vpc: VPC is created if none is provided\n* emrApplicationSecurityGroup: security group created with VPC\n* s3GatewayVpcEndpoint: S3 Gateway endpoint attached to VPC\n\nThe construct is depicted below:\n\n![Spark Runtime Serverless](../../../website/static/img/adsf-spark-runtime.png)\n\n## Usage\n\nThe code snippet below shows a usage example of the `SparkEmrServerlessRuntime` construct.\n\n```ts lit=src/processing/examples/spark-emr-runtime-serverless-default.lit.ts\nclass ExampleSparkEmrServerlessStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n\n const runtimeServerless = new dsf.processing.SparkEmrServerlessRuntime(this, 'SparkRuntimeServerless', {\n name: 'spark-serverless-demo',\n });\n\n const s3ReadPolicyDocument = new PolicyDocument({\n statements: [\n PolicyStatement.fromJson({\n actions: ['s3:GetObject'],\n resources: ['arn:aws:s3:::bucket_name'],\n }),\n ],\n });\n\n // The IAM role that will trigger the Job start and will monitor it\n const jobTrigger = new Role(this, 'EMRServerlessExecutionRole', {\n assumedBy: new ServicePrincipal('lambda.amazonaws.com'),\n });\n\n const executionRole = dsf.processing.SparkEmrServerlessRuntime.createExecutionRole(this, 'EmrServerlessExecutionRole', s3ReadPolicyDocument);\n\n runtimeServerless.grantStartExecution(jobTrigger, executionRole.roleArn);\n\n new cdk.CfnOutput(this, 'SparkRuntimeServerlessStackApplicationArn', {\n value: runtimeServerless.application.attrArn,\n });\n }\n}\n```\n\n\n[//]: # (processing.spark-emr-serverless-job)\n# Spark EMR Serverless job\n\nAn [Amazon EMR Serverless](https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/getting-started.html) Spark job orchestrated through AWS Step Functions state machine.\n\n## Overview\n\nThe construct creates an AWS Step Functions state machine that is used to submit a Spark job and orchestrate the lifecycle of the job. The construct leverages the [AWS SDK service integrations](https://docs.aws.amazon.com/step-functions/latest/dg/supported-services-awssdk.html) to submit the jobs. The state machine can take a cron expression to trigger the job at a given interval. The schema below shows the state machine:\n\n![Spark EMR Serverless Job](../../../website/static/img/adsf-spark-emr-serverless-job.png)\n\n## Usage\n\nThe example stack below shows how to use `EmrServerlessSparkJob` construct. The stack also contains a `SparkEmrServerlessRuntime` to show how to create an EMR Serverless Application and pass it as an argument to the `Spark job` and use it as a runtime for the job.\n\n```ts lit=src/processing/examples/spark-job-emr-serverless.lit.ts\nclass ExampleSparkJobEmrServerlessStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n\n const nightJob = new dsf.processing.SparkEmrServerlessJob(this, 'PiJob', {\n applicationId: runtime.application.attrApplicationId,\n name: 'PiCalculation',\n executionRole: executionRole,\n executionTimeout: cdk.Duration.minutes(15),\n s3LogBucket: Bucket.fromBucketName(this, 'LogBucket', 'emr-job-logs-EXAMPLE'),\n s3LogPrefix: 'logs',\n sparkSubmitEntryPoint: 'local:///usr/lib/spark/examples/src/main/python/pi.py',\n });\n\n new CfnOutput(this, 'job-state-machine', {\n value: nightJob.stateMachine!.stateMachineArn,\n });\n```\n\n\n## Using the EMR Serverless `StartJobRun` parameters\n\nThe `SparkEmrServerlessJobProps` interface provides a simple abstraction to create an EMR Serverless Job. For deeper control on the job configuration, you can also use the `SparkEmrServerlessJobApiProps` inteface which provide the same interface as the [StartJobRun API](https://docs.aws.amazon.com/emr-serverless/latest/APIReference/API_StartJobRun.html) from EMR Serverless.\n\n\n[//]: # (processing.pyspark-application-package)\n# PySpark Application Package\n\nA PySpark application packaged with its dependencies and uploaded on an S3 artifact bucket.\n\n## Overview\n\nThe construct package your PySpark application (the entrypoint, supporting files and virtual environment)\nand upload it to an Amazon S3 bucket. In the rest of the documentation we call the entrypoint,\nsupporting files and virtual environment as artifacts.\n\nThe PySpark Application Package has two responsibilities:\n\n* Upload your PySpark entrypoint application to an artifact bucket\n* Package your PySpark virtual environment (venv) and upload it to an artifact bucket. The package of venv is done using docker,\n an example in the [Usage](#usage) section shows how to write the Dockerfile to package the application.\n\nThe construct uses the [Asset](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_s3_assets.Asset.html)\nto upload the PySpark Application artifacts to CDK Asset bucket. These are then copied to an S3 bucket we call artifact bucket.\n\nTo manage the lifecycle of the artifacts as CDK assets, the constructs need Docker daemon running on the local machine.\nMake sure to have Docker running before using the construct.\n\n### Construct attributes\n\nThe construct exposes the artifacts through the following interfaces:\n\n* entrypointS3Uri: The S3 location where the entry point is saved in S3. You pass this location to your Spark job.\n* venvArchiveS3Uri: The S3 location where the archive of the Python virtual environment with all dependencies is stored. You pass this location to your Spark job.\n* sparkVenvConf: The Spark config containing the configuration of virtual environment archive with all dependencies.\n\n### Resources created\n* An Amazon S3 Bucket to store the PySpark Application artifacts. You can also provide your own if you have already a bucket that you want to use. This bucket comes with configuration to enforce `TLS`, `Block Public Access` and encrypt objects with `SSE-KMS`,\n* An IAM role used by a Lambda to copy from the CDK Asset bucket to the artifact bucket created above or provided.\n\nThe schema below shows the resources created and the responsible of the construct:\n\n![PySpark Application Package](../../../website/static/img/adsf-pyspark-application-package.png)\n\n## Usage\n\nIn this example we will show you how you can use the construct to package a PySpark application\nand submit a job to EMR Serverless leveraging DSF `SparkEmrServerlessRuntime` and `SparkJob` constructs.\n\nFor this example we assume we will have the folder structure as shown below. We have two folders, one containing\nthe `PySpark` application called `spark` folder and a second containing the `CDK` code called `cdk`.\nThe PySpark code, follows the standards `Python` structure. The `spark` also contains the `Dockerfile` to build the `venv`.\nIn the next [section](#dockerfile-definition) will describe how to structure the `Dockerfile`.\n\n```bash\nroot\n|--spark\n| |--test\n| |--src\n| |--__init__.py\n| |--entrypoint.py\n| |--dir1\n| |--__init__.py\n| |--helpers.py\n| |--requirement.txt\n| |--Dockerfile #contains the build instructions to package the virtual environment for PySpark\n|--cdk #contains the CDK application that deploys CDK stack with the PySparkApplicationPackage\n```\n### PySpark Application Definition\n\nFor this example we define the PySparkApplicationPackage resource as follows:\n\n```ts lit=src/processing/examples/pyspark-application-package.lit.ts\nnew dsf.processing.PySparkApplicationPackage(this, 'PySparkApplicationPackage', {\n applicationName: 'nightly-job-aggregation',\n entrypointPath: './../spark/src/entrypoint.py',\n dependenciesFolder: './../spark',\n venvArchivePath: '/venv-package/pyspark-env.tar.gz',\n});\n```\n\n### Dockerfile definition\n\nThe steps below describe how to create the `Dockerfile` so it can be used to be package `venv` by the construct\n\n* In order to build the virtual environment, the docker container will mount the `dependencies_folder`, in our case we define it as `./../spark`.\n* Then to package the `venv` we need to build `COPY` all the files in `./spark` to the docker container.\n* Last we execute the `venv-package`, in the [PySparkApplication](#pyspark-application-definition) we passed the `venv_archive_path` as `/venv-package/pyspark-env.tar.gz`.\n So we need to create it with `mkdir /venv-package` and then pass it to the `venv-package` as `venv-pack -o /venv-package/pyspark-env.tar.gz`\n\n```Dockerfile\nFROM --platform=linux/amd64 public.ecr.aws/amazonlinux/amazonlinux:latest AS base\n\nRUN dnf install -y python3\n\nENV VIRTUAL_ENV=/opt/venv\nRUN python3 -m venv $VIRTUAL_ENV\nENV PATH=\"$VIRTUAL_ENV/bin:$PATH\"\n\nCOPY . .\n\nRUN python3 -m pip install --upgrade pip && \\\n python3 -m pip install venv-pack==0.2.0 && \\\n python3 -m pip install .\n\nRUN mkdir /venv-package && venv-pack -o /venv-package/pyspark-env.tar.gz && chmod ugo+r /venv-package/pyspark-env.tar.gz\n```\n\n### Define a CDK stack upload PySpark application and run the job\n\nThe stack below leverages the resources defined above for PySpark to build the end to end example for building and submitting a PySpark job.\n\n```ts lit=src/processing/examples/pyspark-application-emr-serverless.lit.ts\n const runtime = new dsf.processing.SparkEmrServerlessRuntime(this, 'SparkRuntime', {\n name: 'mySparkRuntime',\n });\n\n const applicationPackage = new dsf.processing.PySparkApplicationPackage(this, 'PySparkApplicationPackage', {\n applicationName: 'nightly-job-aggregation',\n entrypointPath: './../spark/src/entrypoint.py',\n dependenciesFolder: './../spark',\n venvArchivePath: '/venv-package/pyspark-env.tar.gz',\n });\n\n new dsf.processing.SparkEmrServerlessJob(this, 'SparkNightlyJob', {\n applicationId: runtime.application.attrApplicationId,\n name: 'nightly_job',\n executionRole: executionRole,\n executionTimeout: cdk.Duration.minutes(15),\n s3LogBucket: Bucket.fromBucketArn(this, 'LogBucket', 'emr-job-logs-EXAMPLE'),\n s3LogPrefix: 'logs',\n sparkSubmitEntryPoint: applicationPackage.entrypointUri, // use the application package entrypoint\n sparkSubmitParameters: '--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.driver.memory=2G --conf spark.executor.cores=2 {sparkEnvConf}',\n });\n```\n\n\n\n\n[//]: # (processing.spark-cicd-pipeline)\n# Spark CI/CD Pipeline\n\nSelf-mutable CI/CD pipeline for a Spark application based on [Amazon EMR](https://aws.amazon.com/fr/emr/) runtime.\n\n## Overview\n\nThe CI/CD pipeline uses [CDK Pipeline](https://docs.aws.amazon.com/cdk/v2/guide/cdk_pipeline.html) and provisions all the resources needed to implement a CI/CD pipeline for a Spark application on Amazon EMR, including:\n* A CodeCommit repository to host the code\n* A CodePipeline triggered from the main branch of the CodeCommit repository to process the CI/CD tasks\n* A CodeBuild stage to build the CDK assets and run the Spark unit tests\n* A Staging stage to deploy the application stack in the staging environment and run optional integration tests\n* A Production stage to deploy the application stack in the production environment\n\n\n![Spark CI/CD Pipeline](../../../website/static/img/adsf-spark-cicd.png)\n\n## Cross-account deployment\n\nYou can use the same account or optionally use different accounts for CI/CD (where this construct is deployed), staging and production (where the application stack is deployed).\nIf using different accounts, bootstrap staging and production accounts with CDK and add a trust relationship from the CI/CD account:\n```bash\ncdk bootstrap --profile staging \\\naws:/// \\\n--trust \\\n--cloudformation-execution-policies \"POLICY_ARN\"\n```\nMore information is available [here](https://docs.aws.amazon.com/cdk/v2/guide/cdk_pipeline.html#cdk_pipeline_bootstrap)\n\nYou need to also provide the accounts information in the cdk.json in the form of:\n```json\n{\n \"staging\": {\n \"account\": \"\",\n \"region\": \"\"\n },\n \"prod\": {\n \"account\": \"\",\n \"region\": \"\"\n }\n}\n```\n\n## Defining a CDK Stack for the Spark application\n\nThe `SparkCICDPipeline` construct deploys an application stack, which contains your business logic, into staging and production environments.\nThe application stack is a standard CDK stack that you provide. It's expected to be passed via a factory class.\n\nTo do this, implement the `ApplicationStackFactory` and its `createStack()` method.\nThe `createStack()` method needs to return a `Stack` instance within the scope passed to the factory method.\nThis is used to create the application stack within the scope of the CDK Pipeline stage.\n\nThe `CICDStage` parameter is automatically passed by the CDK Pipeline via the factory method and allows you to customize the behavior of the Stack based on the stage.\nFor example, staging stage is used for integration tests so testing a processing job should be done via manually triggering it.\nIn opposition to production stage where the processing job could be automated on a regular basis.\n\nCreate your application stack using the factory pattern:\n\n```ts lit=src/processing/examples/cicd-application-stack.lit.ts\nexport class EmrApplicationStackFactory extends dsf.utils.ApplicationStackFactory {\n createStack(scope: Construct, stage: dsf.utils.CICDStage): cdk.Stack {\n return new EmrApplicationStack(scope, 'EmrApplicationStack', stage);\n }\n}\n\nexport class EmrApplicationStack extends cdk.Stack {\n constructor(scope: Construct, id: string, stage: dsf.utils.CICDStage) {\n super(scope, id);\n\n // DEFINE YOUR APPLICATION STACK HERE\n // USE STAGE PARAMETER TO CUSTOMIZE THE STACK BEHAVIOR\n\n if (stage == dsf.utils.CICDStage.PROD) {\n // prod only\n }\n }\n}\n```\n\nUse the factory to pass your application stack to the `SparkCICDPipeline` construct:\n\n```ts lit=src/processing/examples/cicd-pipeline-stack.lit.ts\nexport class CICDPipelineStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n new dsf.processing.SparkEmrCICDPipeline(this, 'SparkCICDPipeline', {\n sparkApplicationName: 'SparkTest',\n applicationStackFactory: new EmrApplicationStackFactory(),\n });\n }\n}\n```\n\n## Unit tests\nThe construct triggers the unit tests as part of the CI/CD process using the EMR docker image and a fail fast approach.\nThe unit tests are run during the first build step and the entire pipeline stops if the unit tests fail.\n\nUnits tests are expected to be run with `pytest` command after a `pip install .` is run from the Spark root folder configured via `sparkPath`.\n\nIn your Pytest script, use a Spark session with a local master and client mode as the unit tests run in a local EMR docker container:\n```python\nspark = (\n SparkSession.builder.master(\"local[1]\")\n .appName(\"local-tests\")\n .config(\"spark.submit.deployMode\", \"client\")\n .config(\"spark.driver.bindAddress\", \"127.0.0.1\")\n .getOrCreate()\n )\n```\n\n## Integration tests\nYou can optionally run integration tests as part of the CI/CD process using the AWS CLI in a bash script that return `0` exit code if success and `1` if failure.\nThe integration tests are triggered after the deployment of the application stack in the staging environment.\n\nYou can run them via `integTestScript` path that should point to a bash script. For example:\n\n```bash\nroot\n|--spark\n| |--integ.sh\n|--cdk\n```\n\n`integ.sh` is a standard bash script using the AWS CLI to validate the application stack. In the following script example, a Step Function from the application stack is triggered and the result of its execution should be successful:\n```bash\n#!/bin/bash\nEXECUTION_ARN=$(aws stepfunctions start-execution --state-machine-arn $STEP_FUNCTION_ARN | jq -r '.executionArn')\nwhile true\ndo\n STATUS=$(aws stepfunctions describe-execution --execution-arn $EXECUTION_ARN | jq -r '.status')\n if [ $STATUS = \"SUCCEEDED\" ]; then\n exit 0\n elif [ $STATUS = \"FAILED\" ] || [ $STATUS = \"TIMED_OUT\" ] || [ $STATUS = \"ABORTED\" ]; then\n exit 1\n else\n sleep 10\n continue\n fi\ndone\n```\n\nTo use resources that are deployed by the Application Stack like the Step Functions state machine ARN in the previous example:\n1. Create a `CfnOutput` in your application stack with the value of your resource\n\n```ts lit=src/processing/examples/cicd-application-stack-output.lit.ts\nexport class EmrApplicationStack extends cdk.Stack {\n constructor(scope: Construct, id: string, _stage: dsf.utils.CICDStage) {\n super(scope, id);\n\n const processingStateMachine = new StateMachine(this, 'ProcessingStateMachine', {\n // definition ...\n });\n\n new cdk.CfnOutput(this, 'ProcessingStateMachineArn', {\n value: processingStateMachine.stateMachineArn,\n });\n }\n}\n```\n\n2. Pass an environment variable to the `SparkCICDPipeline` construct in the form of a key/value pair via `integTestEnv`:\n * Key is the name of the environment variable used in the script: `STEP_FUNCTION_ARN` in the script example above.\n * Value is the CloudFormation output name from the application stack: `ProcessingStateMachineArn` in the application stack example above.\n * Add permissions required to run the integration tests script. In this example, `states:StartExecution` and `states:DescribeExecution`.\n\n```ts lit=src/processing/examples/cicd-pipeline-stack-tests.lit.ts\nnew dsf.processing.SparkEmrCICDPipeline(this, 'SparkCICDPipeline', {\n sparkApplicationName: 'SparkTest',\n applicationStackFactory: new EmrApplicationStackFactory(),\n integTestScript: 'spark/integ.sh',\n integTestEnv: {\n STEP_FUNCTION_ARN: 'ProcessingStateMachineArn',\n },\n integTestPermissions: [\n new PolicyStatement({\n actions: [\n 'states:StartExecution',\n 'states:DescribeExecution',\n ],\n resources: ['*'],\n }),\n ],\n});\n```\n\n\n\n[//]: # (processing.spark-emr-runtime-containers)\n# Spark EMR Containers Runtime\n\nAn EMR on EKS runtime with preconfigured EKS cluster.\n\n## Overview\n\nThe constructs creates an EKS cluster, install the necessary controllers and enable it to be used by EMR on EKS service as described in this [documentation](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-cluster-access.html). The following are the details of the components deployed.\n\n * An EKS cluster (VPC configuration can be customized)\n * A tooling nodegroup to run various tools including controllers\n * Kubernetes controlers: EBS CSI Driver, Karpenter, ALB Ingress Controller, cert-manager\n * Optional default Kaprenter `NodePools` and `EC2NodeClass` as listed [here](https://github.com/awslabs/data-solutions-framework-on-aws/tree/main/framework/src/processing/lib/spark-runtime/emr-containers/resources/k8s/karpenter-provisioner-config).\n * An S3 bucket to store the pod templates for the `NodePools` created above.\n\nAdditionally, the construct exposes methods to facilitate the creation of EC2 capacity, Virtual Cluster and Execution roles.\n\n## Usage\n\nThe code snippet below shows a usage example of the `SparkEmrContainersRuntime` construct.\n\n```ts lit=src/processing/examples/spark-emr-runtime-containers-default.lit.ts\nclass ExampleSparkEmrContainersStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n\n //Layer must be changed according to the Kubernetes version used\n const kubectlLayer = new KubectlV27Layer(this, 'kubectlLayer');\n\n const emrEksCluster = SparkEmrContainersRuntime.getOrCreate(this, {\n eksAdminRole: Role.fromRoleArn(this, 'EksAdminRole' , 'arn:aws:iam::12345678912:role/role-name-with-path'),\n publicAccessCIDRs: ['10.0.0.0/32'], // The list of public IP addresses from which the cluster can be accessible\n createEmrOnEksServiceLinkedRole: true, //if the the service linked role already exists set this to false\n kubectlLambdaLayer: kubectlLayer,\n });\n\n const s3Read = new PolicyDocument({\n statements: [new PolicyStatement({\n actions: [\n 's3:GetObject',\n ],\n resources: [\n 'arn:aws:s3:::aws-data-analytics-workshop',\n 'arn:aws:s3:::aws-data-analytics-workshop/*'],\n })],\n });\n\n const s3ReadPolicy = new ManagedPolicy(this, 's3ReadPolicy', {\n document: s3Read,\n });\n\n const virtualCluster = emrEksCluster.addEmrVirtualCluster(this, {\n name: 'dailyjob',\n createNamespace: true,\n eksNamespace: 'dailyjobns',\n });\n\n const execRole = emrEksCluster.createExecutionRole(\n this,\n 'ExecRole',\n s3ReadPolicy,\n 'dailyjobns', // the namespace of the virtual cluster\n 's3ReadExecRole'); //the IAM role name\n\n new cdk.CfnOutput(this, 'virtualClusterArn', {\n value: virtualCluster.attrArn,\n });\n\n new cdk.CfnOutput(this, 'execRoleArn', {\n value: execRole.roleArn,\n });\n\n //Driver pod template\n new cdk.CfnOutput(this, 'driverPodTemplate', {\n value: emrEksCluster.podTemplateS3LocationCriticalDriver!,\n });\n\n //Executor pod template\n new cdk.CfnOutput(this, 'executorPodTemplate', {\n value: emrEksCluster.podTemplateS3LocationCriticalExecutor!,\n });\n\n }\n }\n```\n\nThe sceenshot below show how the cloudformation will look like once you deploy the example.\n\n![Spark EMR Containers Cloudfromation Output](../../../website/static/img/emr-eks-construct-cloudformation-output.png)\n\nYou can execute the following command to run a sample job with the infratructure deployed using SparkEmrContainerRuntime:\n\n```sh\naws emr-containers start-job-run \\\n--virtual-cluster-id FROM-CFNOUTPUT-VIRTUAL_CLUSTER_ID \\\n--name spark-pi \\\n--execution-role-arn FROM-CFNOUTPUT-jOB_EXECUTION_ROLE_ARN \\\n--release-label emr-7.0.0-latest \\\n--job-driver '{\n \"sparkSubmitJobDriver\": {\n \"entryPoint\": \"s3://aws-data-analytics-workshops/emr-eks-workshop/scripts/pi.py\",\n \"sparkSubmitParameters\": \"--conf spark.executor.instances=8 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1 --conf spark.kubernetes.driver.podTemplateFile=FROM-CFNOUTPUT-DRIVER-POD-TEMPLATE --conf spark.kubernetes.executor.podTemplateFile=FROM-CFNOUTPUT-EXECUTOR-POD-TEMPLATE\"\n }\n }'\n```\n\n:::warning IAM role requirements\n\nMake sure the role used for running has the correct [IAM policy](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonemroneksemrcontainers.html#amazonemroneksemrcontainers-actions-as-permissions) with `StartJobRun` permission to execute the job.\n\n:::\n\n\n### Isolating workloads on a shared cluster\n\nWith EMR on EKS you can leverage the same EKS cluster to run Spark jobs from multiple teams leveraging namespace segragation through the EMR virtual cluster. The `SparkEMRContainersRuntime` simplifies the creation of an EMR virtual cluster.\nThe `addEmrVirtualCluster()` method enables the EKS cluster to be used by EMR on EKS through the creation EMR on EKS virtual cluster.\nThe method configures the right Kubernetes RBAC as described [here](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-cluster-access.html). It can optionally create the namespace for you.\n\n```ts lit=src/processing/examples/spark-emr-runtime-containers-virtual-cluster.lit.ts\n const virtualCluster = emrEksCluster.addEmrVirtualCluster(this, {\n name: 'dailyjob',\n createNamespace: true,\n eksNamespace: 'dailyjobns',\n });\n\n new cdk.CfnOutput(this, 'virtualClusterArn', {\n value: virtualCluster.attrArn,\n });\n```\n\n### EC2 Capacity\n\nThe EC2 capacity to execute the jobs is defined with [Karpenter](https://karpenter.sh/docs/getting-started/) `NodePools` and `EC2NodeClass`. By default, the construct configure Karpenter `NodePools` and `EC2NodeClass` for 3 types of workloads:\n * Critical workloads\n * Shared workloads\n * Notebook workloads\nYou can opt out from their creation by setting the `default_nodes` to `False`.\n\nTo run EMR on EKS jobs on this EC2 capacity, the construct creates [Pod templates](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/pod-templates.html) and uploads them to an S3 bucket (created by the construct).\nThe pod template are provided for both the Spark driver and the Spark executors and for each of the workload types. They are configured to schedule the Spark pods on the corresponding Karpenter `NodePools` and `EC2NodeClass`.\nThe pod templates locations are stored as class attribute and can be exposed via CloudFormation outputs.\nThe usage example below shows how to provide these as CloudFormation output. The pod templates are referenced in your `spark configuration` that is part of your job defintion.\n\n```ts lit=src/processing/examples/spark-emr-runtime-containers-ec2-capacity.lit.ts\n const emrEksCluster = SparkEmrContainersRuntime.getOrCreate(this, {\n eksAdminRole: Role.fromRoleArn(this, 'EksAdminRole' , 'arn:aws:iam::12345678912:role/role-name-with-path'),\n publicAccessCIDRs: ['10.0.0.0/32'], // The list of public IP addresses from which the cluster can be accessible\n createEmrOnEksServiceLinkedRole: true, //if the the service linked role already exists set this to false\n kubectlLambdaLayer: kubectlLayer,\n defaultNodes: true,\n });\n\n //Driver pod template for critical workloads\n new cdk.CfnOutput(this, 'driverPodTemplate', {\n value: emrEksCluster.podTemplateS3LocationCriticalDriver!,\n });\n\n //Executor pod template for critical workloads\n new cdk.CfnOutput(this, 'executorPodTemplate', {\n value: emrEksCluster.podTemplateS3LocationCriticalExecutor!,\n });\n```\n\nThe construct also exposes the `addKarpenterNodePoolAndNodeClass()` method to define your own EC2 capacity. This method takes a YAML file as defined in [Karpenter](https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/#5-create-nodepool) and apply it to the EKS cluster. You can consult an example [here](https://github.com/awslabs/data-solutions-framework-on-aws/blob/main/framework/src/processing/lib/spark-runtime/emr-containers/resources/k8s/karpenter-provisioner-config/v0.32.1/critical-provisioner.yml).\n\n\n### Execution role\n\nThe execution role is the IAM role that is used by the Spark job to access AWS resources. For example, the job may need to access an S3 bucket that stores the source data or to which the job writes the data. The `createExecutionRole()` method simplifies the creation of an IAM role that can be used to execute a Spark job on the EKS cluster and in a specific EMR EKS virtual cluster namespace. The method attaches an IAM policy provided by the user and a policy to access the pod templates when using the default EC2 capacity.\n\n```ts lit=src/processing/examples/spark-emr-runtime-containers-execrole.lit.ts\n emrEksCluster.addEmrVirtualCluster(this, {\n name: 'dailyjob',\n createNamespace: true,\n eksNamespace: 'dailyjobns',\n });\n\n const execRole = emrEksCluster.createExecutionRole(\n this,\n 'ExecRole',\n s3ReadPolicy, // the IAM managed policy granting permissions required by the Spark job\n 'dailyjobns', // the namespace of the virtual cluster\n 's3ReadExecRole' //the IAM role name\n );\n\n new cdk.CfnOutput(this, 'execRoleArn', {\n value: execRole.roleArn,\n });\n```\n\n\n### Interactive endpoint\n\nThe interactive endpoint provides the capability for interactive clients like Amazon EMR Studio or a self-hosted Jupyter notebook to connect to Amazon EMR on EKS clusters to run interactive workloads. The interactive endpoint is backed by a Jupyter Enterprise Gateway that provides the remote kernel lifecycle management capability that interactive clients need.\n\n```ts lit=src/processing/examples/spark-emr-runtime-containers-interactive-endpoint.lit.ts\n const virtualCluster = emrEksCluster.addEmrVirtualCluster(this, {\n name: 'dailyjob',\n createNamespace: true,\n eksNamespace: 'dailyjobns',\n });\n\n const execRole = emrEksCluster.createExecutionRole(this, 'ExecRole', s3ReadPolicy, 'dailyjobns', 's3ReadExecRole');\n\n const interactiveSession = emrEksCluster.addInteractiveEndpoint(this, 'interactiveSession', {\n virtualClusterId: virtualCluster.attrId,\n managedEndpointName: 'interactiveSession',\n executionRole: execRole,\n });\n\n //Virtual Cluster ARN\n new cdk.CfnOutput(this, 'virtualClusterArn', {\n value: virtualCluster.attrArn,\n });\n\n //Interactive session ARN\n new cdk.CfnOutput(this, 'interactiveSessionArn', {\n value: interactiveSession.getAttString('arn'),\n });\n```\n\n### Grant Job Execution\n\nThe Grant Job Execution allow you to provide an IAM role the rights to start the execution of a job and monitor it in a given virtual cluster. The policy attached will be as follow.\n\n```json\n{\n\t\"Version\": \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Action\": [\n\t\t\t\t\"emr-containers:DescribeJobRun\",\n\t\t\t\t\"emr-containers:ListJobRuns\"\n\t\t\t],\n\t\t\t\"Resource\": \"arn:aws:emr-containers:REGION:ACCOUNT-ID:/virtualclusters/aaabbccmmm\",\n\t\t\t\"Effect\": \"Allow\"\n\t\t},\n\t\t{\n\t\t\t\"Condition\": {\n\t\t\t\t\"ArnEquals\": {\n\t\t\t\t\t\"emr-containers:ExecutionRoleArn\": [\n\t\t\t\t\t\t\"arn:aws:iam::ACCOUNT-ID:role/s3ReadExecRole\"\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"Action\": \"emr-containers:StartJobRun\",\n\t\t\t\"Resource\": \"arn:aws:emr-containers:REGION:ACCOUNT-ID:/virtualclusters/aaabbccmmm\",\n\t\t\t\"Effect\": \"Allow\"\n\t\t},\n\t\t{\n\t\t\t\"Action\": \"emr-containers:TagResource\",\n\t\t\t\"Resource\": \"arn:aws:emr-containers:REGION:ACCOUNT-ID:/virtualclusters/aaabbccmmm/jobruns/*\",\n\t\t\t\"Effect\": \"Allow\"\n\t\t}\n\t]\n}\n```\n\n```ts lit=src/processing/examples/spark-emr-runtime-containers-grant-execution.lit.ts\n // IAM role that is used to start the execution and monitor its state\n const startJobRole: IRole = Role.fromRoleName(this, 'StartJobRole', 'StartJobRole');\n\n SparkEmrContainersRuntime.grantStartJobExecution(startJobRole, [execRole.roleArn], virtualCluster.attrArn);\n```\n" }, "symbolId": "src/processing/index:" }, "aws-dsf.storage": { "locationInModule": { "filename": "src/index.ts", "line": 4 }, "readme": { "markdown": "[//]: # (storage.access-logs-bucket)\n# AccessLogsBucket\n\nAmazon S3 Bucket configured for S3 access logs storage.\n\n## Overview\n\n`AccessLogsBucket` construct is an Amazon S3 Bucket configured with best practices and smart defaults for storing S3 access logs:\n- The default bucket name is in the form of `accesslogs----`\n- The bucket encryption is S3 Managed.\n- Public access is blocked.\n- Two-step protection for bucket and objects deletion.\n- SSL communication is enforced.\n\n`AccessLogsBucket` extends the Amazon [S3 `Bucket`](https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_s3/Bucket.html#bucket) CDK Construct. For custom requirements that are not covered, use the `Bucket` construct directly.\n\n## Usage\n\n```ts lit=src/storage/examples/access-logs-bucket-default.lit.ts\nclass ExampleDefaultAccessLogsBucketStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n new dsf.storage.AccessLogsBucket(this, 'AccessLogsBucket');\n }\n}\n```\n\n## Bucket Naming\n\nThe construct ensures the default bucket name uniqueness which is a [pre-requisite](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html) to create Amazon S3 buckets.\nTo achieve this, the construct is creating the default bucket name like `accesslogs---` where:\n* `` and `` are the account ID and region where you deploy the construct.\n* `` is an 8 characters unique ID calculated based on the CDK path.\n\nIf you provide the `bucketName` parameter, you need to ensure the name is globaly unique.\nAlternatively, you can use the `BucketUtils.generateUniqueBucketName()` utility method to create unique names.\nThis method generates a unique name based on the provided name, the construct ID and the CDK scope:\n* The bucket name is suffixed the AWS account ID, the AWS region and an 8 character hash of the CDK path.\n* The maximum length for the bucket name is 26 characters.\n\n```ts lit=src/storage/examples/access-logs-bucket-naming.lit.ts\nnew dsf.storage.AccessLogsBucket(this, 'MyAccessLogs', {\n bucketName: dsf.utils.BucketUtils.generateUniqueBucketName(this, 'MyAccessLogs', 'my-custom-name')\n});\n```\n\n[//]: # (storage.analytics-bucket)\n# AnalyticsBucket\n\nAmazon S3 Bucket configured for analytics.\n\n## Overview\n\n`AnalyticsBucket` is an Amazon S3 Bucket configured with the following best-practices and defaults for analytics:\n- The bucket name is in the form of `----`\n- Server side bucket encryption managed by KMS customer key. You need to provide a [KMS Key](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html)\n- SSL communication enforcement.\n- Access logged to an S3 bucket within a prefix matching the bucket name. By default, store logs in itself.\n- All public access blocked.\n- Two-step protection for bucket and objects deletion.\n\n`AnalyticsBucket` extends the Amazon [S3 `Bucket`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_s3.Bucket.html) CDK Construct. For custom requirements that are not covered, use the [`Bucket`](https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_s3/Bucket.html#bucket) construct directly.\n\n\n## Usage\n\n```ts lit=src/storage/examples/analytics-bucket-default.lit.ts\nclass ExampleDefaultAnalyticsBucketStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n const key = new Key(this, 'DataKey', {\n enableKeyRotation: true\n });\n\n new dsf.storage.AnalyticsBucket(this, 'AnalyticsBucket', {\n encryptionKey: key\n });\n }\n}\n```\n\n## Objects removal\n\nYou can specify if the bucket and objects should be deleted when the CDK resource is destroyed using `removalPolicy`. Refer to the [DataLakeStorage documentation](https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Storage/data-lake-storage#objects-removal)\n\n:::warning\nIf no access logs bucket is configure, the `AnalyticsBucket` stores its access logs in itself.\nThis prevents from properly deleting the resources when the `removalPolicy` is set to `DESTROY` because it creates an edge case where the deletion of the S3 Objects recreates Objects corresponding the access logs. The S3 Bucket is then never empty and the deletion of the Bucket fails.\n:::\n\n## Bucket Naming\n\nThe construct ensures the default bucket name uniqueness which is a [pre-requisite](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html) to create Amazon S3 buckets.\nTo achieve this, the construct is creating the default bucket name like `accesslogs---` where:\n* `` and `` are the account ID and region where you deploy the construct.\n* `` is an 8 characters unique ID calculated based on the CDK path.\n\nIf you provide the `bucketName` parameter, you need to ensure the name is globaly unique.\nAlternatively, you can use the `BucketUtils.generateUniqueBucketName()` utility method to create unique names.\nThis method generates a unique name based on the provided name, the construct ID and the CDK scope:\n* The bucket name is suffixed the AWS account ID, the AWS region and an 8 character hash of the CDK path.\n* The maximum length for the bucket name is 26 characters.\n\n```ts lit=src/storage/examples/analytics-bucket-naming.lit.ts\nnew dsf.storage.AnalyticsBucket(this, 'AnalyticsBucket', {\n bucketName: dsf.utils.BucketUtils.generateUniqueBucketName(this, 'AnalyticsBucket', 'my-custom-name'),\n encryptionKey: key\n});\n```\n\n[//]: # (storage.data-lake-storage)\n# DataLakeStorage\n\nData Lake based on medallion architecture and AWS best-practices.\n\n## Overview\n\nAt the high level, `DataLakeStorage` creates three [Amazon S3](https://aws.amazon.com/s3) buckets configured specifically for data lake on AWS. By default these buckets are named *Bronze*, *Silver*, and *Gold* to represent [different data layers](https://docs.aws.amazon.com/prescriptive-guidance/latest/defining-bucket-names-data-lakes/data-layer-definitions.html). You can customize bucket names according to your needs.\n`DataLakeStorage` uses [`AnalyticsBucket`](analytics-bucket) and [`AccessLogsBucket`](access-logs-bucket) constructs from DSF on AWS, to create storage and access logs buckets respectively. Your data lake storage is encrypted using [AWS KMS](https://aws.amazon.com/kms/) a default customer managed key. You can also provide your own KMS Key. We provide data lifecycle management that you can customize to your needs.\n\nHere is the overview of `DataLakeStorage` features:\n- Medalion design with S3 buckets for Bronze, Silver, and Gold data.\n- Server-side encryption using a single KMS customer key for all S3 buckets.\n- Enforced SSL in-transit encryption.\n- Logs data lake access in a dedicated bucket within a prefix matching the bucket name.\n- Buckets, objects and encryption keys can be retained when the CDK resource is destroyed (default).\n- All public access blocked.\n\n![Data lake storage](../../../website/static/img/adsf-data-lake-storage.png)\n\n## Usage\n\n```ts lit=src/storage/examples/data-lake-storage-default.lit.ts\nclass ExampleDefaultDataLakeStorageStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n new dsf.storage.DataLakeStorage(this, 'MyDataLakeStorage');\n }\n}\n```\n\n## Bucket naming\n\nThe construct ensures the default bucket names uniqueness which is a [pre-requisite](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html) to create Amazon S3 buckets.\nTo achieve this, the construct is creating the default bucket names like `---` where:\n* `` is the layer in the medallion architecture (bronze, silver or gold).\n* `` and `` are the account ID and region where you deploy the construct.\n* `` is an 8 characters unique ID calculated based on the CDK path.\n\nIf you provide the `bucketName` parameter, you need to ensure the name is globaly unique.\nAlternatively, you can use the `BucketUtils.generateUniqueBucketName()` utility method to create unique names.\nThis method generates a unique name based on the provided name, the construct ID and the CDK scope:\n* The bucket name is suffixed the AWS account ID, the AWS region and an 8 character hash of the CDK path.\n* The maximum length for the bucket name is 26 characters.\n\n```ts lit=src/storage/examples/data-lake-storage-naming.lit.ts\nnew dsf.storage.DataLakeStorage(this, 'MyDataLakeStorage', {\n bronzeBucketName: dsf.utils.BucketUtils.generateUniqueBucketName(this, 'MyDataLakeStorage', 'custom-bronze-name')\n});\n```\n\n## Objects removal\n\nYou can specify if buckets, objects and encryption keys should be deleted when the CDK resource is destroyed using `removalPolicy`. To have an additional layer of protection, we require users to set a global context value for data removal in their CDK applications.\n\nBuckets, objects and encryption keys can be destroyed when the CDK resource is destroyed only if **both** data lake removal policy and DSF on AWS global removal policy are set to remove objects.\n\nYou can set `@data-solutions-framework-on-aws/removeDataOnDestroy` (`true` or `false`) global data removal policy in `cdk.json`:\n\n```json title=\"cdk.json\"\n{\n \"context\": {\n \"@data-solutions-framework-on-aws/removeDataOnDestroy\": true\n }\n}\n```\n\nOr programmatically in your CDK app:\n\n```ts lit=src/storage/examples/data-lake-storage-objects-removal.lit.ts\n // Set context value for global data removal policy\n this.node.setContext('@data-solutions-framework-on-aws/removeDataOnDestroy', true);\n\n new dsf.storage.DataLakeStorage(this, 'DataLakeStorage', {\n removalPolicy: RemovalPolicy.DESTROY\n });\n```\n\n\n## Data lifecycle management\nWe provide a simple [data lifecycle management](https://aws.amazon.com/s3/storage-classes/) for data lake storage, that you can customize to your needs. By default:\n- Bronze data is moved to Infrequent Access after 30 days and archived to Glacier after 90 days.\n- Silver and Gold data is moved to Infrequent Access after 90 days and is not archived.\n\nChange the data lifecycle rules using the DataLakeStorage properties:\n\n```ts lit=src/storage/examples/data-lake-storage-lifecycle.lit.ts\nnew dsf.storage.DataLakeStorage(this, 'MyDataLakeStorage', {\n bronzeBucketInfrequentAccessDelay: 90,\n bronzeBucketArchiveDelay: 180,\n silverBucketInfrequentAccessDelay: 180,\n silverBucketArchiveDelay: 360,\n goldBucketInfrequentAccessDelay: 180,\n goldBucketArchiveDelay: 360,\n});\n```\n" }, "symbolId": "src/storage/index:" }, "aws-dsf.utils": { "locationInModule": { "filename": "src/index.ts", "line": 7 }, "readme": { "markdown": "[//]: # (utils.s3-data-copy)\n# S3DataCopy\n\nData copy from one bucket to another during deployment time.\n\n## Overview\n\n`S3DataCopy` construct provides a process to copy objects from one bucket to another during CDK deployment time:\n- The copy is part of the CDK and CloudFormation deployment process. It's using a synchronous CDK Custom Resource running on AWS Lambda.\n- The Lambda function is written in Typescript and copies objects between source and target buckets.\n- The execution role used by the Lambda function is scoped to the least privileges. A custom role can be provided.\n- The Lambda function can be executed in an Amazon VPC within private subnets. By default, it runs run inside VPCs owned by the AWS Lambda service.\n- The Lambda function is granted read access on the source bucket and write access on the destination bucket using the execution role policy. The construct doesn't grant cross account access.\n\n![S3 Data Copy](../../../website/static/img/s3-data-copy.png)\n\n## Usage\n\n```ts lit=src/utils/examples/s3-data-copy.lit.ts\nclass ExampleDefaultS3DataCopyStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n\n const sourceBucket = Bucket.fromBucketName(this, 'sourceBucket', 'nyc-tlc');\n const targetBucket = Bucket.fromBucketName(this, 'destinationBucket', 'staging-bucket');\n\n new dsf.utils.S3DataCopy(this, 'S3DataCopy', {\n sourceBucket,\n sourceBucketPrefix: 'trip data/',\n sourceBucketRegion: 'us-east-1',\n targetBucket,\n targetBucketPrefix: 'staging-data/',\n });\n }\n}\n```\n\n## Private networks\n\nThe lambda Function used by the custom resource can be deployed in a VPC by passing the VPC and a list of private subnets.\n\nPublic subnets are not supported.\n\n```ts lit=src/utils/examples/s3-data-copy-vpc.lit.ts\n const vpc = Vpc.fromLookup(this, 'Vpc', { vpcName: 'my-vpc'});\n const subnets = vpc.selectSubnets({subnetType: SubnetType.PRIVATE_WITH_EGRESS});\n\n new dsf.utils.S3DataCopy(this, 'S3DataCopy', {\n sourceBucket,\n sourceBucketPrefix: 'trip data/',\n sourceBucketRegion: 'us-east-1',\n targetBucket,\n targetBucketPrefix: 'staging-data/',\n vpc,\n subnets,\n });\n```\n\n[//]: # (utils.data-vpc)\n# DataVpc\n\nAmazon VPC optimized for data solutions.\n\n## Overview\n\n`DataVpc` construct provides a standard Amazon VPC with best practices for security and data solutions implementations:\n- The VPC is created with public and private subnets across 3 availability zones (1 of each per AZ) and 3 NAT gateways.\n- VPC CIDR mask should be larger than 28. The CIDR is split between public and private subnets with private subnets being twice as large as public subnet.\n- The flow logs maaged by a dedicated least-privilege IAM Role. The role can be customized.\n- The flow logs exported to an Amazon CloudWatch LogGroup encrypted with an Amazon KMS customer managed key. The KMS key can be customized.\n- A gateway VPC endpoint is created for S3 access.\n\n## Usage\n\n```ts lit=src/utils/examples/data-vpc-default.lit.ts\nclass ExampleDefaultDataVpcStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n new dsf.utils.DataVpc(this, 'MyDataVpc', {\n vpcCidr: '10.0.0.0/16',\n });\n }\n}\n```\n\n\n## VPC Flow Logs\n\nThe construct logs VPC Flow logs in a Cloudwatch Log Group that is encrypted with a customer managed KMS Key. Exporting VPC Flow Logs to CloudWatch requires an IAM Role.\nYou can customize the VPC Flow Logs management with:\n- your own KMS Key. Be sure to attach the right permissions to your key.\nRefer to the [AWS documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) for full description.\n- your own IAM Role. Be sure to configure the proper trust policy and permissions. Refer to the [AWS documentation](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-cwl.html#flow-logs-iam-role) for full description.\n- a custom log retention policy. Default is one week.\n\n```ts lit=src/utils/examples/data-vpc-flowlog.lit.ts\n const flowLogKey = Key.fromKeyArn(this, 'FlowLogKey', 'XXXXXXXXXXXXXXXXXXXXXXXX');\n\n const flowLogRole = Role.fromRoleArn(this, 'FlowLogRole', 'XXXXXXXXXXXXXXXXXXXXXXXX');\n\n new dsf.utils.DataVpc(this, 'MyDataVpc', {\n vpcCidr: '10.0.0.0/16',\n flowLogKey,\n flowLogRole,\n flowLogRetention: RetentionDays.TWO_WEEKS,\n });\n }\n}\n```\n\n## Removal policy\n\nYou can specify if the Cloudwatch Log Group and the KMS encryption Key should be deleted when the CDK resource is destroyed using `removalPolicy`. To have an additional layer of protection, we require users to set a global context value for data removal in their CDK applications.\n\nLog group and encryption key can be destroyed when the CDK resource is destroyed only if **both** data vpc removal policy and DSF on AWS global removal policy are set to remove objects.\n\nYou can set `@data-solutions-framework-on-aws/removeDataOnDestroy` (`true` or `false`) global data removal policy in `cdk.json`:\n\n```json title=\"cdk.json\"\n{\n \"context\": {\n \"@data-solutions-framework-on-aws/removeDataOnDestroy\": true\n }\n}\n```\n\nOr programmatically in your CDK app:\n\n```ts lit=src/utils/examples/data-vpc-removal.lit.ts\n // Set context value for global data removal policy\n this.node.setContext('@data-solutions-framework-on-aws/removeDataOnDestroy', true);\n\n new dsf.utils.DataVpc(this, 'MyDataVpc', {\n vpcCidr: '10.0.0.0/16',\n removalPolicy: RemovalPolicy.DESTROY\n });\n }\n}\n```\n\n[//]: # (utils.customization)\n# Customize DSF on AWS constructs\n\nYou can customize DSF on AWS constructs in several ways to adapt to your specific needs:\n1. Use the Constructs properties instead of the smart defaults.\n2. Extend existing constructs and override specific methods or properties.\n3. Access CDK L1 resources and override any property.\n\n\n## Constructs properties\n\nUse the properties of the construct to adapt the behavior to your needs. With this approach, you bypass the smart defaults provided by the construct.\nRefer to the documentation of each construct to evaluate if your requirements can be implemented.\n\n:::note\nThis method should always be preferred because constructs properties are tested as part of the DSF on AWS build process.\n:::\n\nFor example, you can use the `DataLakeStorage` properties to modify the lifecycle configurations for transitioning objects based on your needs instead of using the default rules:\n\n```ts lit=src/storage/examples/data-lake-storage-lifecycle.lit.ts\nnew dsf.storage.DataLakeStorage(this, 'MyDataLakeStorage', {\n bronzeBucketInfrequentAccessDelay: 90,\n bronzeBucketArchiveDelay: 180,\n silverBucketInfrequentAccessDelay: 180,\n silverBucketArchiveDelay: 360,\n goldBucketInfrequentAccessDelay: 180,\n goldBucketArchiveDelay: 360,\n});\n```\n\n## Construct extension\n\nAWS CDK allows developers to extend classes like any object-oriented programing language. You can use this method when you want to:\n* Override a specific method exposed by a construct.\n* Implement your own defaults. Refer to the example of the [`AnalyticsBucket`](https://github.com/awslabs/data-solutions-framework-on-aws/blob/main/framework/src/storage/analytics-bucket.ts) that extends the CDK L2 `Bucket` construct to enforce some of the parameters.\n\n## CDK resources override\n\nAWS CDK offers escape hatches to modify constructs that are encapsulated in DSF on AWS constructs. The constructs always expose the AWS resources that are encapsulated so you can manually modify their configuration. For achieving this you have 3 options:\n\n* Modify the L2 construct using its CDK API. For example, you can modify the buckets' policies provided by the [`DataLakeStorage`](https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/data-lake-storage) to provide cross account write access. All the [buckets](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_s3.Bucket.html) L2 constructs are exposed as an object parameter:\n\n```ts lit=src/utils/examples/customization-l2.lit.ts\n // Create a data lake using DSF on AWS L3 construct\n const storage = new dsf.storage.DataLakeStorage(this, 'MyDataLakeStorage');\n\n // Access the CDK L2 Bucket construct exposed by the L3 construct\n const goldBucket = storage.goldBucket;\n\n // Use the Bucket CDK API to modify the Bucket Policy and add cross account write access\n goldBucket.addToResourcePolicy(new aws_iam.PolicyStatement({\n actions: [\n 's3:GetObject',\n 's3:PutObject',\n 's3:DeleteObject',\n 's3:ListBucketMultipartUploads',\n 's3:ListMultipartUploadParts',\n 's3:AbortMultipartUpload',\n 's3:ListBucket',\n ],\n effect: aws_iam.Effect.ALLOW,\n principals: [new aws_iam.AccountPrincipal('123456789012')]\n }\n ));\n```\n\n* [Modify the L1 construct resource](https://docs.aws.amazon.com/cdk/v2/guide/cfn_layer.html#cfn_layer_resource) when there is a CDK property available on the L1 construct.\n For example, you can override CDK L1 property for setting the S3 transfer Acceleration on the gold bucket of the `DataLakeStorage`:\n\n```ts lit=src/utils/examples/customization-l1.lit.ts\n // Create a data lake using DSF on AWS L3 construct\n const storage = new dsf.storage.DataLakeStorage(this, 'MyDataLakeStorage');\n\n // Access the CDK L1 Bucket construct exposed by the L3 construct\n const cfnBucket = storage.goldBucket.node.defaultChild as CfnBucket;\n\n // Override the CDK L1 property for transfer acceleration\n cfnBucket.accelerateConfiguration = {\n accelerationStatus: 'Enabled',\n }\n```\n\n* [Override the CloudFormation properties](https://docs.aws.amazon.com/cdk/v2/guide/cfn_layer.html#cfn_layer_raw) when there isn't any CDK property, and you need to modify the CFN template directly.\n For example, you can override CloudFormation property for setting the S3 transfer Acceleration on the gold bucket of the `DataLakeStorage`:\n\n```ts lit=src/utils/examples/customization-cfn.lit.ts\n // Create a data lake using DSF on AWS L3 construct\n const storage = new dsf.storage.DataLakeStorage(this, 'MyDataLakeStorage');\n\n // Access the CDK L1 Bucket construct exposed by the L3 construct\n const cfnBucket = storage.goldBucket.node.defaultChild as CfnBucket;\n\n // Override the CloudFormation property for transfer acceleration\n cfnBucket.addOverride('Properties.AccelerateConfiguration.AccelerationStatus', 'Enabled')\n```\n\n[//]: # (utils.custom-resources)\n# Create custom resources with the DsfProvider\n\nDSF provides an internal construct named `DsfProvider` to facilitate the creation of custom resources in DSF constructs.\nThe `DsfProvider` construct handles the undifferentiated tasks for you so you can focus on the custom resource logic.\nThis construct is an opinionated implementation of the [CDK Custom Resource Provider Framework](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib-readme.html#the-custom-resource-provider-framework).\nIt creates:\n* A custom resource provider to manage the entire custom resource lifecycle\n* An onEvent Lambda function from the provided code to perform actions you need in your custom resource\n* An optional isComplete Lambda function from the provided code when using [asynchronous custom resources](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.custom_resources-readme.html#asynchronous-providers-iscomplete)\n* CloudWatch Logs log groups for each Lambda function\n* IAM roles for each Lambda function and associated permissions\n\n:::note\nYou still need to provide an IAM Managed Policy required by the actions of the Lambda functions.\n:::\n\n## Configuring handlers for the custom resource\n\nThe `DsfProvider` construct requires a Lambda function handler called `onEvent` to perform the actions of the custom resource. It also supports an optional Lambda function handler called `isComplete` to regularly perform status checks for asynchronous operation triggered in the `onEvent` handler.\n\nBoth Lambda functions are implemented in Typescript.\n`esbuild` is used to package the Lambda code and is automatically installed by `Projen`. If `esbuild` is available, `docker` will be used.\nYou need to configure the path of the Lambda code (entry file) and the path of the dependency lock file (`package-lock.json`) for each handler.\n\nTo generate the `package-lock.json` file, run from the Lambda code folder:\n\n```bash\nnpm install --package-lock-only\n```\n\nThen you can configure the `onEvent` and `isComplete` handlers in the `DsfProvider` construct:\n\n```ts lit=src/utils/examples/dsf-provider-is-complete.lit.ts\nimport { DsfProvider } from '../lib/dsf-provider';\n\nclass ExampleIsCompleteDsfProviderStack extends cdk.Stack{\n constructor(scope: Construct, id: string) {\n super(scope, id);\n\n const myProvider = new DsfProvider(this, 'Provider', {\n providerName: 'my-provider',\n onEventHandlerDefinition: {\n managedPolicy: myOnEventManagedPolicy,\n handler: 'on-event.handler',\n depsLockFilePath: path.join(__dirname, './resources/lambda/my-cr/package-lock.json'),\n entryFile: path.join(__dirname, './resources/lambda/my-cr/on-event.mjs'),\n },\n isCompleteHandlerDefinition: {\n managedPolicy: myIsCompleteManagedPolicy,\n handler: 'is-complete.handler',\n depsLockFilePath: path.join(__dirname, './resources/lambda/my-cr/package-lock.json'),\n entryFile: path.join(__dirname, './resources/lambda/my-cr/is-complete.mjs'),\n }\n });\n\n new cdk.CustomResource(this, 'CustomResource', {\n serviceToken: myProvider.serviceToken,\n resourceType: 'Custom::MyCustomResource',\n });\n }\n}\n```\n\n## Packaging dependencies in the Lambda function\n\nDependencies can be added to the Lambda handlers using the bundling options. For example, the following code adds the AWS SDK S3 client to the `onEvent` handler:\n\n```ts lit=src/utils/examples/dsf-provider-bundling.lit.ts\nconst myProvider = new DsfProvider(this, 'Provider', {\n providerName: 'my-provider',\n onEventHandlerDefinition: {\n managedPolicy: myManagedPolicy,\n handler: 'on-event.handler',\n depsLockFilePath: path.join(__dirname, './resources/lambda/my-cr/package-lock.json'),\n entryFile: path.join(__dirname, './resources/lambda/my-cr/on-event.mjs'),\n bundling: {\n nodeModules: [\n '@aws-sdk/client-s3',\n ],\n commandHooks: {\n afterBundling: () => [],\n beforeBundling: () => [\n 'npx esbuild --version'\n ],\n beforeInstall: () => [\n ]\n }\n },\n },\n});\n```\n\n## Running the Custom Resource in VPC\n\nYou can configure the `DsfProvider` to run all the Lambda functions within a VPC (for example in private subnets). It includes the Lambda handlers (`onEvent` and `isComplete`) and the Lambda functions used by the custom resource framework. The following configurations are available when running the custom resource in a VPC:\n * The VPC where you want to run the custom resource.\n * The subnets where you want to run the Lambda functions. Subnets are optional. If not configured, the construct uses the VPC default strategy to select subnets.\n * The EC2 security groups to attach to the Lambda functions. Security groups are optional. If not configured, a single security group is created for all the Lambda functions.\n\n:::danger\nThe `DsfProvider` construct implements a custom process to efficiently clean up ENIs when deleting the custom resource. Without this process it can take up to one hour to delete the ENI and dependant resources.\nThis process requires the security groups to be dedicated to the custom resource. If you configure security groups, ensure they are dedicated.\n:::\n\n```ts lit=src/utils/examples/dsf-provider-vpc.lit.ts\n\n const vpc = Vpc.fromLookup(this, 'Vpc', { vpcName: 'my-vpc'});\n const subnets = vpc.selectSubnets({subnetType: SubnetType.PRIVATE_WITH_EGRESS});\n const securityGroup = SecurityGroup.fromSecurityGroupId(this, 'SecurityGroup', 'sg-123456');\n\n const myProvider = new DsfProvider(this, 'Provider', {\n providerName: 'my-provider',\n onEventHandlerDefinition: {\n managedPolicy: myManagedPolicy,\n handler: 'on-event.handler',\n depsLockFilePath: path.join(__dirname, './resources/lambda/my-cr/package-lock.json'),\n entryFile: path.join(__dirname, './resources/lambda/my-cr/on-event.mjs'),\n },\n vpc,\n subnets,\n // the security group should be dedicated to the custom resource\n securityGroups: [securityGroup],\n });\n```\n\n## Configuring environment variables of Lambda handlers\n\nLambda handlers can leverage environment variables to pass values to the Lambda code. You can configure environment variables for each of the Lambda handlers:\n\n```ts lit=src/utils/examples/dsf-provider-env.lit.ts\nconst myProvider = new DsfProvider(this, 'Provider', {\n providerName: 'my-provider',\n onEventHandlerDefinition: {\n managedPolicy: myManagedPolicy,\n handler: 'on-event.handler',\n depsLockFilePath: path.join(__dirname, './resources/lambda/my-cr/package-lock.json'),\n entryFile: path.join(__dirname, './resources/lambda/my-cr/on-event.mjs'),\n environment: {\n MY_ENV_VARIABLE: 'my-env-variable-value',\n }\n },\n});\n```\n\n## Removal policy\n\nYou can specify if the Cloudwatch Log Groups should be deleted when the CDK resource is destroyed using `removalPolicy`. To have an additional layer of protection, we require users to set a global context value for data removal in their CDK applications.\n\nLog groups can be destroyed when the CDK resource is destroyed only if **both** `DsfProvider` removal policy and DSF on AWS global removal policy are set to remove objects.\n\nYou can set `@data-solutions-framework-on-aws/removeDataOnDestroy` (`true` or `false`) global data removal policy in `cdk.json`:\n\n```json title=\"cdk.json\"\n{\n \"context\": {\n \"@data-solutions-framework-on-aws/removeDataOnDestroy\": true\n }\n}\n```\n\nOr programmatically in your CDK app:\n\n```ts lit=src/utils/examples/dsf-provider-removal-policy.lit.ts\n\n this.node.setContext('@data-solutions-framework-on-aws/removeDataOnDestroy', true);\n\n const myProvider = new DsfProvider(this, 'Provider', {\n providerName: 'my-provider',\n onEventHandlerDefinition: {\n managedPolicy: myOnEventManagedPolicy,\n handler: 'on-event.handler',\n depsLockFilePath: path.join(__dirname, './resources/lambda/my-cr/package-lock.json'),\n entryFile: path.join(__dirname, './resources/lambda/my-cr/on-event.mjs'),\n },\n removalPolicy: cdk.RemovalPolicy.DESTROY,\n });\n```\n" }, "symbolId": "src/utils/index:" } }, "targets": { "js": { "npm": "aws-dsf" }, "python": { "distName": "aws_dsf", "module": "aws_dsf" } }, "types": { "aws-dsf.consumption.AthenaWorkGroup": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "new dsf.consumption.AthenaWorkGroup(this, 'AthenaWorkGroupDefault', {\n name: 'athena-default',\n resultLocationPrefix: 'athena-default-results/'\n })", "stability": "stable", "summary": "An Amazon Athena Workgroup configured with default result bucket." }, "fqn": "aws-dsf.consumption.AthenaWorkGroup", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup.ts", "line": 54 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "fqn": "aws-dsf.consumption.AthenaWorkgroupProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup.ts", "line": 28 }, "methods": [ { "docs": { "stability": "stable", "summary": "Grants running queries access to Principal." }, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup.ts", "line": 227 }, "name": "grantRunQueries", "parameters": [ { "docs": { "summary": "Principal to attach query access to Athena Workgroup." }, "name": "principal", "type": { "fqn": "aws-cdk-lib.aws_iam.IPrincipal" } } ] }, { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "AthenaWorkGroup", "namespace": "consumption", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "S3 Bucket used for query results." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup.ts", "line": 40 }, "name": "resultBucket", "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "docs": { "stability": "stable", "summary": "KMS Key to encrypt the query results." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup.ts", "line": 44 }, "name": "resultsEncryptionKey", "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "docs": { "stability": "stable", "summary": "Athena Workgroup that is created." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup.ts", "line": 32 }, "name": "workGroup", "type": { "fqn": "aws-cdk-lib.aws_athena.CfnWorkGroup" } }, { "docs": { "stability": "stable", "summary": "WorkGroup name with the randomized suffix." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup.ts", "line": 36 }, "name": "workGroupName", "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "Role used to access user resources in an Athena for Apache Spark session." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup.ts", "line": 48 }, "name": "executionRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } } ], "symbolId": "src/consumption/lib/athena/athena-workgroup:AthenaWorkGroup" }, "aws-dsf.consumption.AthenaWorkgroupProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for the AthenaWorkgroup Construct." }, "fqn": "aws-dsf.consumption.AthenaWorkgroupProps", "kind": "interface", "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 14 }, "name": "AthenaWorkgroupProps", "namespace": "consumption", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "Name of the Workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 18 }, "name": "name", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "Specifies the location in Amazon S3 where query results are stored." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 73 }, "name": "resultLocationPrefix", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "Indicates the number of days after creation when objects are deleted from the Result bucket." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 95 }, "name": "bytesScannedCutoffPerQuery", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- True.", "stability": "stable", "summary": "If set to \"true\", the settings for the workgroup override client-side settings." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 48 }, "name": "enforceWorkGroupConfiguration", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- AUTO.", "stability": "stable", "summary": "The engine version on which the query runs." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 36 }, "name": "engineVersion", "optional": true, "type": { "fqn": "aws-dsf.consumption.EngineVersion" } }, { "abstract": true, "docs": { "default": "- The role is created if PySpark engine version is selected and no role is provided.", "stability": "stable", "summary": "Role used to access user resources in an Athena for Apache Spark session." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 85 }, "name": "executionRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "default": "- True.", "stability": "stable", "summary": "Indicates that the Amazon CloudWatch metrics are enabled for the workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 79 }, "name": "publishCloudWatchMetricsEnabled", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- Workgroup is retained.", "stability": "stable", "summary": "The option to delete a workgroup and its contents even if the workgroup contains any named queries." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 24 }, "name": "recursiveDeleteOption", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 56 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- False.", "stability": "stable", "summary": "Allows members assigned to a workgroup to reference Amazon S3 Requester Pays buckets in queries." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 42 }, "name": "requesterPaysEnabled", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- Create a new bucket with SSE encryption using AnalyticsBucket if not provided.", "stability": "stable", "summary": "Amazon S3 Bucket where query results are stored." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 62 }, "name": "resultBucket", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "abstract": true, "docs": { "default": "- Name will be provided.", "stability": "stable", "summary": "Name for the S3 Bucket in case it should be created." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 68 }, "name": "resultBucketName", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- The key is created if Result Bucket is not provided.", "remarks": "Has to be provided if Result bucket is provided.\nUser needs to grant access to it for AthenaWorkGroup's executionRole (if Spark engine) or for\nprincipals that were granted to run queries using AthenaWorkGroup's grantRunQueries.", "stability": "stable", "summary": "Encryption key used to encrypt query results." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 103 }, "name": "resultsEncryptionKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "Indicates the number of days after creation when objects are deleted from the Result bucket." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 90 }, "name": "resultsRetentionPeriod", "optional": true, "type": { "fqn": "aws-cdk-lib.Duration" } }, { "abstract": true, "docs": { "default": "- ENABLED.", "stability": "stable", "summary": "The state of the Workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-props.ts", "line": 30 }, "name": "state", "optional": true, "type": { "fqn": "aws-dsf.consumption.State" } } ], "symbolId": "src/consumption/lib/athena/athena-workgroup-props:AthenaWorkgroupProps" }, "aws-dsf.consumption.EngineVersion": { "assembly": "aws-dsf", "docs": { "stability": "stable" }, "fqn": "aws-dsf.consumption.EngineVersion", "kind": "enum", "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-config.ts", "line": 4 }, "members": [ { "docs": { "stability": "stable" }, "name": "AUTO" }, { "docs": { "stability": "stable" }, "name": "ATHENA_V3" }, { "docs": { "stability": "stable" }, "name": "PYSPARK_V3" } ], "name": "EngineVersion", "namespace": "consumption", "symbolId": "src/consumption/lib/athena/athena-workgroup-config:EngineVersion" }, "aws-dsf.consumption.RedshiftData": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "const namespace = new dsf.consumption.RedshiftServerlessNamespace(this, 'RedshiftNamespace', {\n name: \"default\",\n dbName: 'defaultdb',\n});\n\nconst workgroup = new dsf.consumption.RedshiftServerlessWorkgroup(this, \"RedshiftWorkgroup\", {\n name: \"redshift-workgroup\",\n namespace: namespace,\n});\n\nconst rsData = workgroup.accessData('DataApi');\nrsData.createDbRole(\"EngineeringRole\", \"defaultdb\", \"engineering\");", "remarks": "If `vpc` and `vpcSubnets` are passed, this construct would also create the Redshift Data Interface VPC endpoint and configure the custom resource in the same VPC subnet.", "stability": "stable", "summary": "Creates an asynchronous custom resource that handles the execution of SQL using Redshift's Data API." }, "fqn": "aws-dsf.consumption.RedshiftData", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 104 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "fqn": "aws-dsf.consumption.RedshiftDataProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 33 }, "methods": [ { "docs": { "stability": "stable", "summary": "Assigns Redshift DB roles to IAM role vs the `RedshiftDbRoles` tag." }, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 324 }, "name": "assignDbRolesToIAMRole", "parameters": [ { "docs": { "summary": "List of Redshift DB roles to assign to IAM role." }, "name": "dbRoles", "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "array" } } }, { "docs": { "summary": "The IAM role to assign the Redshift DB roles to." }, "name": "targetRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } } ] }, { "docs": { "returns": "`CustomResource`", "stability": "stable", "summary": "Creates a new DB role." }, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 280 }, "name": "createDbRole", "parameters": [ { "docs": { "summary": "The CDK Construct ID." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "The name of the database to run this command." }, "name": "databaseName", "type": { "primitive": "string" } }, { "docs": { "summary": "The name of the role to create." }, "name": "roleName", "type": { "primitive": "string" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.CustomResource" } } }, { "docs": { "returns": "`CustomResource`", "stability": "stable", "summary": "Grants both read and write permissions on all the tables in the `schema` to the DB role." }, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 315 }, "name": "grantDbAllPrivilegesToRole", "parameters": [ { "docs": { "summary": "The CDK Construct ID." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "The name of the database to run this command." }, "name": "databaseName", "type": { "primitive": "string" } }, { "docs": { "summary": "The schema where the tables are located in." }, "name": "schema", "type": { "primitive": "string" } }, { "docs": { "summary": "The DB role to grant the permissions to." }, "name": "roleName", "type": { "primitive": "string" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.CustomResource" } } }, { "docs": { "returns": "`CustomResource`", "stability": "stable", "summary": "Grants access to the schema to the DB role." }, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 292 }, "name": "grantDbSchemaToRole", "parameters": [ { "docs": { "summary": "The CDK Construct ID." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "The name of the database to run this command." }, "name": "databaseName", "type": { "primitive": "string" } }, { "docs": { "summary": "The schema where the tables are located in." }, "name": "schema", "type": { "primitive": "string" } }, { "docs": { "summary": "The DB role to grant the permissions to." }, "name": "roleName", "type": { "primitive": "string" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.CustomResource" } } }, { "docs": { "returns": "`CustomResource`", "stability": "stable", "summary": "Grants read permission on all the tables in the `schema` to the DB role." }, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 303 }, "name": "grantSchemaReadToRole", "parameters": [ { "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "The name of the database to run this command." }, "name": "databaseName", "type": { "primitive": "string" } }, { "docs": { "summary": "The schema where the tables are located in." }, "name": "schema", "type": { "primitive": "string" } }, { "docs": { "summary": "The DB role to grant the permissions to." }, "name": "roleName", "type": { "primitive": "string" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.CustomResource" } } }, { "docs": { "returns": "", "stability": "stable", "summary": "Ingest data from S3 into a Redshift table." }, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 340 }, "name": "ingestData", "parameters": [ { "docs": { "summary": "The CDK Construct ID." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "The name of the database to run this command." }, "name": "databaseName", "type": { "primitive": "string" } }, { "docs": { "summary": "The target table to load the data into." }, "name": "targetTable", "type": { "primitive": "string" } }, { "docs": { "summary": "The bucket where the source data would be coming from." }, "name": "sourceBucket", "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "docs": { "summary": "The location inside the bucket where the data would be ingested from." }, "name": "sourcePrefix", "type": { "primitive": "string" } }, { "docs": { "remarks": "Additional options to pass to the `COPY` command. For example, `delimiter '|'` or `ignoreheader 1`", "summary": "Optional." }, "name": "ingestAdditionalOptions", "optional": true, "type": { "primitive": "string" } }, { "docs": { "remarks": "The IAM Role to use to access the data in S3. If not provided, it would use the default IAM role configured in the Redshift Namespace", "summary": "Optional." }, "name": "role", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.CustomResource" } } }, { "docs": { "remarks": "This command would do an upsert into the target table.", "returns": "`CustomResource`", "stability": "stable", "summary": "Run the `MERGE` query using simplified mode." }, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 367 }, "name": "mergeToTargetTable", "parameters": [ { "docs": { "summary": "The CDK Construct ID." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "The name of the database to run this command." }, "name": "databaseName", "type": { "primitive": "string" } }, { "docs": { "remarks": "Schema can also be included using the following format: `schemaName.tableName`", "summary": "The source table name." }, "name": "sourceTable", "type": { "primitive": "string" } }, { "docs": { "remarks": "Schema can also be included using the following format: `schemaName.tableName`", "summary": "The target table name." }, "name": "targetTable", "type": { "primitive": "string" } }, { "docs": { "remarks": "Default is `id`", "summary": "The column in the source table that's used to determine whether the rows in the `sourceTable` can be matched with rows in the `targetTable`." }, "name": "sourceColumnId", "optional": true, "type": { "primitive": "string" } }, { "docs": { "remarks": "Default is `id`", "summary": "The column in the target table that's used to determine whether the rows in the `sourceTable` can be matched with rows in the `targetTable`." }, "name": "targetColumnId", "optional": true, "type": { "primitive": "string" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.CustomResource" } } }, { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } }, { "docs": { "remarks": "Once the custom resource finishes execution, the attribute `Data` contains an attribute `execId` which contains the Redshift Data API execution ID. You can then use this to retrieve execution results via the `GetStatementResult` API.", "returns": "`CustomResource`", "stability": "stable", "summary": "Runs a custom SQL." }, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 257 }, "name": "runCustomSQL", "parameters": [ { "docs": { "summary": "The CDK Construct ID." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "The name of the database to run this command." }, "name": "databaseName", "type": { "primitive": "string" } }, { "docs": { "summary": "The sql to run." }, "name": "sql", "type": { "primitive": "string" } }, { "docs": { "remarks": "The sql to run when this resource gets deleted", "summary": "Optional." }, "name": "deleteSql", "optional": true, "type": { "primitive": "string" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.CustomResource" } } } ], "name": "RedshiftData", "namespace": "consumption", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The IAM Role for the Redshift Data API execution." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 46 }, "name": "executionRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The Lambda Function for the Redshift Data API status checks." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 55 }, "name": "statusFunction", "type": { "fqn": "aws-cdk-lib.aws_lambda.IFunction" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group for the Redshift Data API status checks." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 51 }, "name": "statusLogGroup", "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The Lambda Function for the Redshift Data submission." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 42 }, "name": "submitFunction", "type": { "fqn": "aws-cdk-lib.aws_lambda.IFunction" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group for the Redshift Data API submission." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 38 }, "name": "submitLogGroup", "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The managed IAM policy allowing IAM Role to retrieve tag information." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 83 }, "name": "taggingManagedPolicy", "type": { "fqn": "aws-cdk-lib.aws_iam.IManagedPolicy" } }, { "docs": { "stability": "stable", "summary": "The ARN of the target cluster or workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 73 }, "name": "targetArn", "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The ID of the target cluster or workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 78 }, "name": "targetId", "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The Lambda function for the S3 data copy cleaning up lambda." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 64 }, "name": "cleanUpFunction", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_lambda.IFunction" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group for the Redshift Data cleaning up lambda." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 60 }, "name": "cleanUpLogGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The IAM Role for the the S3 data copy cleaning up lambda." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 68 }, "name": "cleanUpRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The Security Group used by the Custom Resource when deployed in a VPC." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 93 }, "name": "customResourceSecurityGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.ISecurityGroup" } }, { "docs": { "stability": "stable", "summary": "The created Redshift Data API interface vpc endpoint when deployed in a VPC." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 88 }, "name": "vpcEndpoint", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.InterfaceVpcEndpoint" } }, { "docs": { "stability": "stable", "summary": "The Security Group used by the VPC Endpoint when deployed in a VPC." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data.ts", "line": 98 }, "name": "vpcEndpointSecurityGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.ISecurityGroup" } } ], "symbolId": "src/consumption/lib/redshift/redshift-data:RedshiftData" }, "aws-dsf.consumption.RedshiftDataProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "The properties for the `RedshiftData` construct." }, "fqn": "aws-dsf.consumption.RedshiftDataProps", "kind": "interface", "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 12 }, "name": "RedshiftDataProps", "namespace": "consumption", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The Secrets Manager Secret containing the admin credentials for the Redshift cluster / namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 29 }, "name": "secret", "type": { "fqn": "aws-cdk-lib.aws_secretsmanager.ISecret" } }, { "abstract": true, "docs": { "default": "- The `workgroupId` is used", "remarks": "It must be configured if the `workgroupId` is not.", "stability": "stable", "summary": "The name of the Redshift provisioned to query." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 18 }, "name": "clusterId", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- false", "stability": "stable", "summary": "If set to true, create the Redshift Data Interface VPC Endpoint in the configured VPC/Subnets." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 55 }, "name": "createInterfaceVpcEndpoint", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- 5mins", "stability": "stable", "summary": "The timeout for the query execution." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 61 }, "name": "executionTimeout", "optional": true, "type": { "fqn": "aws-cdk-lib.Duration" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise, the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 69 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- no secret key is used", "stability": "stable", "summary": "The KMS Key used to encrypt the admin credentials for the Redshift cluster / namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 35 }, "name": "secretKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "default": "- No subnets are used. The Custom Resource runs in the Redshift service team subnets.", "remarks": "A Redshift Data API Interface VPC Endpoint is created in the subnets.", "stability": "stable", "summary": "The subnets where the Custom Resource Lambda Function would be created in." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 49 }, "name": "subnets", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.SelectedSubnets" } }, { "abstract": true, "docs": { "default": "- No VPC is used. The Custom Resource runs in the Redshift service team VPC", "remarks": "A Redshift Data API Interface VPC Endpoint is created in the VPC.", "stability": "stable", "summary": "The VPC where the Custom Resource Lambda Function would be created in." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 42 }, "name": "vpc", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.IVpc" } }, { "abstract": true, "docs": { "default": "- The `clusterId` is used", "remarks": "It must be configured if the `clusterId` is not.", "stability": "stable", "summary": "The `workgroupId` for the Redshift Serverless Workgroup to query." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift/redshift-data-props.ts", "line": 24 }, "name": "workgroupId", "optional": true, "type": { "primitive": "string" } } ], "symbolId": "src/consumption/lib/redshift/redshift-data-props:RedshiftDataProps" }, "aws-dsf.consumption.RedshiftServerlessNamespace": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "const namespace = new dsf.consumption.RedshiftServerlessNamespace(this, 'DefaultServerlessNamespace', {\n dbName: 'defaultdb',\n name: 'default'\n});", "stability": "stable", "summary": "Create a Redshift Serverless Namespace with the admin credentials stored in Secrets Manager." }, "fqn": "aws-dsf.consumption.RedshiftServerlessNamespace", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 96 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "fqn": "aws-dsf.consumption.RedshiftServerlessNamespaceProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 24 }, "methods": [ { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "RedshiftServerlessNamespace", "namespace": "consumption", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The created Secrets Manager secret containing the admin credentials." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 61 }, "name": "adminSecret", "type": { "fqn": "aws-cdk-lib.aws_secretsmanager.ISecret" } }, { "docs": { "stability": "stable", "summary": "The KMS Key used to encrypt the admin credentials secret." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 65 }, "name": "adminSecretKey", "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "docs": { "stability": "stable", "summary": "The Lambda Function for the Redshift Serverless creation." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 37 }, "name": "createFunction", "type": { "fqn": "aws-cdk-lib.aws_lambda.IFunction" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Logs Log Group for the Redshift Serverless creation." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 33 }, "name": "createLogGroup", "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The IAM Role for the Redshift Serverless creation." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 41 }, "name": "createRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The custom resource that creates the Namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 29 }, "name": "customResource", "type": { "fqn": "aws-cdk-lib.CustomResource" } }, { "docs": { "stability": "stable", "summary": "KMS key used by the namespace to encrypt the data." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 75 }, "name": "dataKey", "type": { "fqn": "aws-cdk-lib.aws_kms.Key" } }, { "docs": { "stability": "stable", "summary": "The name of the database." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 79 }, "name": "dbName", "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The ARN of the created namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 83 }, "name": "namespaceArn", "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The ID of the created namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 87 }, "name": "namespaceId", "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The name of the created namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 57 }, "name": "namespaceName", "type": { "primitive": "string" } }, { "docs": { "remarks": "These roles are used to access other AWS services for ingestion, federated query, and data catalog access.", "see": "https://docs.aws.amazon.com/redshift/latest/mgmt/redshift-iam-authentication-access-control.html", "stability": "stable", "summary": "The roles attached to the namespace in the form of `{RoleArn: IRole}`." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 71 }, "name": "roles", "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_iam.IRole" }, "kind": "map" } } }, { "docs": { "stability": "stable", "summary": "The Lambda Function for the creation status check." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 49 }, "name": "statusFunction", "type": { "fqn": "aws-cdk-lib.aws_lambda.IFunction" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Logs Log Group for the creation status check." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 45 }, "name": "statusLogGroup", "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The IAM Role for the creation status check." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace.ts", "line": 53 }, "name": "statusRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } } ], "symbolId": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace:RedshiftServerlessNamespace" }, "aws-dsf.consumption.RedshiftServerlessNamespaceLogExport": { "assembly": "aws-dsf", "docs": { "stability": "stable", "summary": "Namespace log export types." }, "fqn": "aws-dsf.consumption.RedshiftServerlessNamespaceLogExport", "kind": "enum", "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 10 }, "members": [ { "docs": { "stability": "stable" }, "name": "USER_LOG" }, { "docs": { "stability": "stable" }, "name": "CONNECTION_LOG" }, { "docs": { "stability": "stable" }, "name": "USER_ACTIVITY_LOG" } ], "name": "RedshiftServerlessNamespaceLogExport", "namespace": "consumption", "symbolId": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props:RedshiftServerlessNamespaceLogExport" }, "aws-dsf.consumption.RedshiftServerlessNamespaceProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "RedshiftServerlessNamespace properties." }, "fqn": "aws-dsf.consumption.RedshiftServerlessNamespaceProps", "kind": "interface", "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 19 }, "name": "RedshiftServerlessNamespaceProps", "namespace": "consumption", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The name of the primary database that would be created in the Redshift Serverless Namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 28 }, "name": "dbName", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The name of the Redshift Serverless Namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 23 }, "name": "name", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- A new KMS Key is created", "stability": "stable", "summary": "The KMS Key used by the managed Secrets Manager Secret storing admin credentials." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 65 }, "name": "adminSecretKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.Key" } }, { "abstract": true, "docs": { "default": "- The default username is \"admin\"", "stability": "stable", "summary": "The admin username to be used." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 59 }, "name": "adminUsername", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- A new KMS Key is created", "stability": "stable", "summary": "The KMS Key used to encrypt the data." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 47 }, "name": "dataKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.Key" } }, { "abstract": true, "docs": { "default": "- No default IAM Role is associated with the Redshift Serverless Namespace", "stability": "stable", "summary": "Default IAM Role associated to the Redshift Serverless Namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 34 }, "name": "defaultIAMRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "default": "- No IAM roles are associated with the Redshift Serverless Namespace", "remarks": "This list of Roles must also contain the `defaultIamRole`.", "stability": "stable", "summary": "List of IAM Roles attached to the Redshift Serverless Namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 41 }, "name": "iamRoles", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_iam.IRole" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- No logs are exported", "stability": "stable", "summary": "The type of logs to be exported." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 53 }, "name": "logExports", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-dsf.consumption.RedshiftServerlessNamespaceLogExport" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise, the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props.ts", "line": 73 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } } ], "symbolId": "src/consumption/lib/redshift-serverless/redshift-serverless-namespace-props:RedshiftServerlessNamespaceProps" }, "aws-dsf.consumption.RedshiftServerlessWorkgroup": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "const workgroup = new dsf.consumption.RedshiftServerlessWorkgroup(this, \"RedshiftWorkgroup\", {\n name: \"example-workgroup\",\n namespace: new dsf.consumption.RedshiftServerlessNamespace(this, \"RedshiftNamespace\", {\n name: 'example-namespace',\n dbName: 'defaultdb',\n })\n});", "remarks": "A default namespace would be created if none is provided.", "stability": "stable", "summary": "Create a Redshift Serverless Workgroup." }, "fqn": "aws-dsf.consumption.RedshiftServerlessWorkgroup", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 67 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "fqn": "aws-dsf.consumption.RedshiftServerlessWorkgroupProps" } } ] }, "interfaces": [ "aws-cdk-lib.aws_ec2.IConnectable" ], "kind": "class", "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 28 }, "methods": [ { "docs": { "returns": "`RedshiftData`", "stability": "stable", "summary": "Creates an instance of `RedshiftData` to send custom SQLs to the workgroup." }, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 174 }, "name": "accessData", "parameters": [ { "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "if set to true, create interface VPC endpoint for Redshift Data API." }, "name": "createVpcEndpoint", "optional": true, "type": { "primitive": "boolean" } } ], "returns": { "type": { "fqn": "aws-dsf.consumption.RedshiftData" } } }, { "docs": { "default": "`/public/%``", "returns": "The DataCatalogDatabase created", "stability": "stable", "summary": "Creates a new Glue data catalog database with a crawler using JDBC target type to connect to the Redshift Workgroup." }, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 117 }, "name": "catalogTables", "parameters": [ { "docs": { "summary": "The CDK ID of the resource." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "The name of the Glue Database to create." }, "name": "catalogDbName", "type": { "primitive": "string" } }, { "docs": { "summary": "The path of Redshift tables to crawl." }, "name": "pathToCrawl", "optional": true, "type": { "primitive": "string" } } ], "returns": { "type": { "fqn": "aws-dsf.governance.DataCatalogDatabase" } } }, { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "RedshiftServerlessWorkgroup", "namespace": "consumption", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The created Redshift Serverless Workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 36 }, "name": "cfnResource", "type": { "fqn": "aws-cdk-lib.aws_redshiftserverless.CfnWorkgroup" } }, { "docs": { "remarks": "Used this to enable access from clients connecting to the workgroup", "stability": "stable", "summary": "Connections used by Workgroup security group." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 40 }, "name": "connections", "overrides": "aws-cdk-lib.aws_ec2.IConnectable", "type": { "fqn": "aws-cdk-lib.aws_ec2.Connections" } }, { "docs": { "remarks": "This can be used by Glue ETL Jobs to read/write data from/to Redshift workgroup", "stability": "stable", "summary": "The Glue Connection associated with the workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 48 }, "name": "glueConnection", "type": { "fqn": "aws-cdk-lib.aws_glue.CfnConnection" } }, { "docs": { "stability": "stable", "summary": "The associated Redshift Serverless Namespace." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 44 }, "name": "namespace", "type": { "fqn": "aws-dsf.consumption.RedshiftServerlessNamespace" } }, { "docs": { "stability": "stable", "summary": "The primary EC2 Security Group associated with the Redshift Serverless Workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 52 }, "name": "primarySecurityGroup", "type": { "fqn": "aws-cdk-lib.aws_ec2.ISecurityGroup" } }, { "docs": { "stability": "stable", "summary": "The subnets where the Redshift Serverless Workgroup is deployed." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 60 }, "name": "selectedSubnets", "type": { "fqn": "aws-cdk-lib.aws_ec2.SelectedSubnets" } }, { "docs": { "stability": "stable", "summary": "The VPC where the Redshift Serverless Workgroup is deployed." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup.ts", "line": 56 }, "name": "vpc", "type": { "fqn": "aws-cdk-lib.aws_ec2.IVpc" } } ], "symbolId": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup:RedshiftServerlessWorkgroup" }, "aws-dsf.consumption.RedshiftServerlessWorkgroupProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "RedshiftServerlessWorkgroup properties." }, "fqn": "aws-dsf.consumption.RedshiftServerlessWorkgroupProps", "kind": "interface", "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props.ts", "line": 11 }, "name": "RedshiftServerlessWorkgroupProps", "namespace": "consumption", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The name of the Redshift Serverless Workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props.ts", "line": 15 }, "name": "name", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The Redshift Serverless Namespace associated with the Workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props.ts", "line": 26 }, "name": "namespace", "type": { "fqn": "aws-dsf.consumption.RedshiftServerlessNamespace" } }, { "abstract": true, "docs": { "default": "- 128 RPU", "stability": "stable", "summary": "The base capacity of the Redshift Serverless Workgroup in RPU." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props.ts", "line": 21 }, "name": "baseCapacity", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- No extra security groups are used", "stability": "stable", "summary": "The extra EC2 Security Groups to associate with the Redshift Serverless Workgroup (in addition to the primary Security Group)." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props.ts", "line": 51 }, "name": "extraSecurityGroups", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_ec2.SecurityGroup" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- 5439", "remarks": "Valid port ranges are 5431-5455 and 8191-8215.", "stability": "stable", "summary": "The custom port to use when connecting to workgroup." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props.ts", "line": 57 }, "name": "port", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise, the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props.ts", "line": 34 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- Use the private subnets of the VPC", "stability": "stable", "summary": "The subnets where the Redshift Serverless Workgroup is deployed." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props.ts", "line": 45 }, "name": "subnets", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.SubnetSelection" } }, { "abstract": true, "docs": { "default": "- A default VPC is created", "stability": "stable", "summary": "The VPC where the Redshift Serverless Workgroup is deployed." }, "immutable": true, "locationInModule": { "filename": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props.ts", "line": 39 }, "name": "vpc", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.Vpc" } } ], "symbolId": "src/consumption/lib/redshift-serverless/redshift-serverless-workgroup-props:RedshiftServerlessWorkgroupProps" }, "aws-dsf.consumption.State": { "assembly": "aws-dsf", "docs": { "stability": "stable" }, "fqn": "aws-dsf.consumption.State", "kind": "enum", "locationInModule": { "filename": "src/consumption/lib/athena/athena-workgroup-config.ts", "line": 10 }, "members": [ { "docs": { "stability": "stable" }, "name": "ENABLED" }, { "docs": { "stability": "stable" }, "name": "DISABLED" } ], "name": "State", "namespace": "consumption", "symbolId": "src/consumption/lib/athena/athena-workgroup-config:State" }, "aws-dsf.governance.DataCatalogDatabase": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "import { Bucket } from 'aws-cdk-lib/aws-s3';\n\nnew dsf.governance.DataCatalogDatabase(this, 'ExampleDatabase', {\n locationBucket: new Bucket(scope, 'LocationBucket'),\n locationPrefix: '/databasePath',\n name: 'example-db'\n});", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Governance/data-catalog-database", "stability": "stable", "summary": "An AWS Glue Data Catalog Database configured with the location and a crawler." }, "fqn": "aws-dsf.governance.DataCatalogDatabase", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/governance/lib/data-catalog-database.ts", "line": 55 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "fqn": "aws-dsf.governance.DataCatalogDatabaseProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/governance/lib/data-catalog-database.ts", "line": 25 }, "methods": [ { "docs": { "remarks": "This would attach an IAM Policy to the principal allowing read access to the Glue Database and all its Glue Tables.", "returns": "`AddToPrincipalPolicyResult`", "stability": "stable", "summary": "Grants read access via identity based policy to the principal." }, "locationInModule": { "filename": "src/governance/lib/data-catalog-database.ts", "line": 236 }, "name": "grantReadOnlyAccess", "parameters": [ { "docs": { "summary": "Principal to attach the Glue Database read access to." }, "name": "principal", "type": { "fqn": "aws-cdk-lib.aws_iam.IPrincipal" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.aws_iam.AddToPrincipalPolicyResult" } } }, { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "DataCatalogDatabase", "namespace": "governance", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The Glue Database that's created." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database.ts", "line": 41 }, "name": "database", "type": { "fqn": "aws-cdk-lib.aws_glue.CfnDatabase" } }, { "docs": { "stability": "stable", "summary": "The Glue Database name with the randomized suffix to prevent name collisions in the catalog." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database.ts", "line": 45 }, "name": "databaseName", "type": { "primitive": "string" } }, { "docs": { "remarks": "This property can be undefined if `autoCrawl` is set to `false`.", "stability": "stable", "summary": "The Glue Crawler created when `autoCrawl` is set to `true` (default value)." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database.ts", "line": 29 }, "name": "crawler", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_glue.CfnCrawler" } }, { "docs": { "stability": "stable", "summary": "KMS encryption Key used by the Crawler." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database.ts", "line": 49 }, "name": "crawlerLogEncryptionKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "docs": { "stability": "stable", "summary": "The IAM Role used by the Glue crawler when created." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database.ts", "line": 37 }, "name": "crawlerRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The Glue security configuration used by the Glue Crawler when created." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database.ts", "line": 33 }, "name": "crawlerSecurityConfiguration", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_glue.CfnSecurityConfiguration" } } ], "symbolId": "src/governance/lib/data-catalog-database:DataCatalogDatabase" }, "aws-dsf.governance.DataCatalogDatabaseProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for the `DataCatalogDatabase` construct." }, "fqn": "aws-dsf.governance.DataCatalogDatabaseProps", "kind": "interface", "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 14 }, "name": "DataCatalogDatabaseProps", "namespace": "governance", "properties": [ { "abstract": true, "docs": { "remarks": "Construct would add a randomize suffix as part of the name to prevent name collisions.", "stability": "stable", "summary": "Database name." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 18 }, "name": "name", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- True", "stability": "stable", "summary": "When enabled, this automatically creates a top level Glue Crawler that would run based on the defined schedule in the `autoCrawlSchedule` parameter." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 54 }, "name": "autoCrawl", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- `cron(1 0 * * ? *)`", "remarks": "Default is once a day at 00:01h.", "stability": "stable", "summary": "The schedule to run the Glue Crawler." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 67 }, "name": "autoCrawlSchedule", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_glue.CfnCrawler.ScheduleProperty" } }, { "abstract": true, "docs": { "default": "- Create a new key if none is provided", "stability": "stable", "summary": "KMS encryption Key used for the Glue Crawler logs." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 73 }, "name": "crawlerLogEncryptionKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "default": "- When `autoCrawl` is enabled, a new role is created with least privilege permissions to run the crawler", "remarks": "Additional permissions are granted to this role such as S3 Bucket read only permissions and KMS encrypt/decrypt on the key used by the Glue Crawler logging to CloudWatch Logs.", "stability": "stable", "summary": "The IAM Role used by the Glue Crawler when `autoCrawl` is set to `True`." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 61 }, "name": "crawlerRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "default": "- calculated based on `locationPrefix`", "remarks": "This helps the Glue Crawler understand the layout of the folders in S3.", "stability": "stable", "summary": "Directory depth where the table folders are located." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 79 }, "name": "crawlerTableLevelDepth", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The connection that would be used by the crawler." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 33 }, "name": "glueConnectionName", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The JDBC path that would be included by the crawler." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 48 }, "name": "jdbcPath", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The secret associated with the JDBC connection." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 38 }, "name": "jdbcSecret", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_secretsmanager.ISecret" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The KMS key used by the JDBC secret." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 43 }, "name": "jdbcSecretKMSKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "S3 bucket where data is stored." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 23 }, "name": "locationBucket", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "Top level location wwhere table data is stored." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 28 }, "name": "locationPrefix", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-catalog-database-props.ts", "line": 87 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } } ], "symbolId": "src/governance/lib/data-catalog-database-props:DataCatalogDatabaseProps" }, "aws-dsf.governance.DataLakeCatalog": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "import { Key } from 'aws-cdk-lib/aws-kms';\n\nconst logEncryptionKey = new Key(this, 'ExampleLogKey');\nconst storage = new dsf.storage.DataLakeStorage(this, \"ExampleStorage\");\nconst dataLakeCatalog = new dsf.governance.DataLakeCatalog(this, \"ExampleDataLakeCatalog\", {\n dataLakeStorage: storage,\n databaseName: \"exampledb\",\n crawlerLogEncryptionKey: logEncryptionKey\n})", "remarks": "The Data Lake Catalog is composed of 3 `DataCatalogDatabase`, one for each storage layer.", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Governance/data-lake-catalog", "stability": "stable", "summary": "Creates a Data Lake Catalog on top of a `DataLakeStorage`." }, "fqn": "aws-dsf.governance.DataLakeCatalog", "initializer": { "docs": { "stability": "stable", "summary": "Constructs a new instance of DataLakeCatalog." }, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog.ts", "line": 52 }, "parameters": [ { "docs": { "summary": "the Scope of the CDK Construct." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the ID of the CDK Construct." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "the DataLakeCatalog properties." }, "name": "props", "type": { "fqn": "aws-dsf.governance.DataLakeCatalogProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/governance/lib/data-lake-catalog.ts", "line": 28 }, "methods": [ { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "DataLakeCatalog", "namespace": "governance", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The Glue Database for the Bronze S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog.ts", "line": 32 }, "name": "bronzeCatalogDatabase", "type": { "fqn": "aws-dsf.governance.DataCatalogDatabase" } }, { "docs": { "stability": "stable", "summary": "The Glue Database for the Gold S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog.ts", "line": 40 }, "name": "goldCatalogDatabase", "type": { "fqn": "aws-dsf.governance.DataCatalogDatabase" } }, { "docs": { "stability": "stable", "summary": "The Glue Database for the Silver S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog.ts", "line": 36 }, "name": "silverCatalogDatabase", "type": { "fqn": "aws-dsf.governance.DataCatalogDatabase" } }, { "docs": { "stability": "stable", "summary": "The KMS Key used to encrypt the Glue Crawler logs." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog.ts", "line": 44 }, "name": "crawlerLogEncryptionKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } } ], "symbolId": "src/governance/lib/data-lake-catalog:DataLakeCatalog" }, "aws-dsf.governance.DataLakeCatalogProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for the `DataLakeCatalog` Construct." }, "fqn": "aws-dsf.governance.DataLakeCatalogProps", "kind": "interface", "locationInModule": { "filename": "src/governance/lib/data-lake-catalog-props.ts", "line": 12 }, "name": "DataLakeCatalogProps", "namespace": "governance", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The DataLakeStorage object to create the data catalog on." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog-props.ts", "line": 17 }, "name": "dataLakeStorage", "type": { "fqn": "aws-dsf.storage.DataLakeStorage" } }, { "abstract": true, "docs": { "default": "- True", "stability": "stable", "summary": "When enabled, creates a top level Glue Crawler that would run based on the defined schedule in the `autoCrawlSchedule` parameter." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog-props.ts", "line": 30 }, "name": "autoCrawl", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- `cron(1 0 * * ? *)`", "remarks": "Default is once a day at 00:01h.", "stability": "stable", "summary": "The schedule when the Glue Crawler runs, if enabled." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog-props.ts", "line": 36 }, "name": "autoCrawlSchedule", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_glue.CfnCrawler.ScheduleProperty" } }, { "abstract": true, "docs": { "default": "- Create a new KMS Key if none is provided", "stability": "stable", "summary": "The KMS encryption Key used for the Glue Crawler logs." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog-props.ts", "line": 42 }, "name": "crawlerLogEncryptionKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "default": "- calculated based on `locationPrefix`", "remarks": "This helps the Glue Crawler understand the layout of the folders in S3.", "stability": "stable", "summary": "Directory depth where the table folders are located." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog-props.ts", "line": 48 }, "name": "crawlerTableLevelDepth", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- Use the bucket name as the database name and as the S3 location", "remarks": "The name of the Glue Database is composed of the S3 Bucket name and this suffix.\nThe suffix is also added to the S3 location inside the data lake S3 Buckets.", "stability": "stable", "summary": "The suffix of the Glue Data Catalog Database." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog-props.ts", "line": 24 }, "name": "databaseName", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/governance/lib/data-lake-catalog-props.ts", "line": 56 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } } ], "symbolId": "src/governance/lib/data-lake-catalog-props:DataLakeCatalogProps" }, "aws-dsf.processing.EmrContainersRuntimeVersion": { "assembly": "aws-dsf", "docs": { "stability": "stable" }, "fqn": "aws-dsf.processing.EmrContainersRuntimeVersion", "kind": "enum", "locationInModule": { "filename": "src/processing/lib/emr-releases.ts", "line": 30 }, "members": [ { "docs": { "stability": "stable" }, "name": "V7_0" }, { "docs": { "stability": "stable" }, "name": "V6_15" }, { "docs": { "stability": "stable" }, "name": "V6_14" }, { "docs": { "stability": "stable" }, "name": "V6_13" }, { "docs": { "stability": "stable" }, "name": "V6_12" }, { "docs": { "stability": "stable" }, "name": "V6_11_1" }, { "docs": { "stability": "stable" }, "name": "V6_11" }, { "docs": { "stability": "stable" }, "name": "V6_10_1" }, { "docs": { "stability": "stable" }, "name": "V6_10" }, { "docs": { "stability": "stable" }, "name": "V6_9" }, { "docs": { "stability": "stable" }, "name": "V6_8" }, { "docs": { "stability": "stable" }, "name": "V6_7" }, { "docs": { "stability": "stable" }, "name": "V6_6" }, { "docs": { "stability": "stable" }, "name": "V6_5" }, { "docs": { "stability": "stable" }, "name": "V6_4" }, { "docs": { "stability": "stable" }, "name": "V6_3" }, { "docs": { "stability": "stable" }, "name": "V6_2" }, { "docs": { "stability": "stable" }, "name": "V5_33" }, { "docs": { "stability": "stable" }, "name": "V5_32" } ], "name": "EmrContainersRuntimeVersion", "namespace": "processing", "symbolId": "src/processing/lib/emr-releases:EmrContainersRuntimeVersion" }, "aws-dsf.processing.EmrRuntimeVersion": { "assembly": "aws-dsf", "docs": { "stability": "stable", "summary": "Enum defining the EMR version as defined in the [Amazon EMR documentation](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-release-components.html)." }, "fqn": "aws-dsf.processing.EmrRuntimeVersion", "kind": "enum", "locationInModule": { "filename": "src/processing/lib/emr-releases.ts", "line": 8 }, "members": [ { "docs": { "stability": "stable" }, "name": "V7_0" }, { "docs": { "stability": "stable" }, "name": "V6_15" }, { "docs": { "stability": "stable" }, "name": "V6_14" }, { "docs": { "stability": "stable" }, "name": "V6_13" }, { "docs": { "stability": "stable" }, "name": "V6_12" }, { "docs": { "stability": "stable" }, "name": "V6_11_1" }, { "docs": { "stability": "stable" }, "name": "V6_11" }, { "docs": { "stability": "stable" }, "name": "V6_10_1" }, { "docs": { "stability": "stable" }, "name": "V6_10" }, { "docs": { "stability": "stable" }, "name": "V6_9" }, { "docs": { "stability": "stable" }, "name": "V6_8" }, { "docs": { "stability": "stable" }, "name": "V6_7" }, { "docs": { "stability": "stable" }, "name": "V6_6" }, { "docs": { "stability": "stable" }, "name": "V6_5" }, { "docs": { "stability": "stable" }, "name": "V6_4" }, { "docs": { "stability": "stable" }, "name": "V6_3" }, { "docs": { "stability": "stable" }, "name": "V6_2" }, { "docs": { "stability": "stable" }, "name": "V5_33" }, { "docs": { "stability": "stable" }, "name": "V5_32" } ], "name": "EmrRuntimeVersion", "namespace": "processing", "symbolId": "src/processing/lib/emr-releases:EmrRuntimeVersion" }, "aws-dsf.processing.EmrVirtualClusterProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "The properties for the `EmrVirtualCluster` Construct class." }, "fqn": "aws-dsf.processing.EmrVirtualClusterProps", "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/emr-virtual-cluster-props.ts", "line": 7 }, "name": "EmrVirtualClusterProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The name of the Amazon EMR Virtual Cluster to be created." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/emr-virtual-cluster-props.ts", "line": 11 }, "name": "name", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- Do not create the namespace", "stability": "stable", "summary": "The flag to create EKS namespace." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/emr-virtual-cluster-props.ts", "line": 22 }, "name": "createNamespace", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- Use the default namespace", "stability": "stable", "summary": "The name of the EKS namespace to be linked to the EMR virtual cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/emr-virtual-cluster-props.ts", "line": 16 }, "name": "eksNamespace", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- none", "stability": "stable", "summary": "The tags assigned to the Virtual Cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/emr-virtual-cluster-props.ts", "line": 29 }, "name": "tags", "optional": true, "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "map" } } } ], "symbolId": "src/processing/lib/spark-runtime/emr-containers/emr-virtual-cluster-props:EmrVirtualClusterProps" }, "aws-dsf.processing.KarpenterVersion": { "assembly": "aws-dsf", "docs": { "stability": "stable", "summary": "The list of supported Karpenter versions as defined [here](https://github.com/aws/karpenter/releases) At this time only v0.32.1 is supported." }, "fqn": "aws-dsf.processing.KarpenterVersion", "kind": "enum", "locationInModule": { "filename": "src/processing/lib/karpenter-releases.ts", "line": 8 }, "members": [ { "docs": { "stability": "stable" }, "name": "V0_32_1" } ], "name": "KarpenterVersion", "namespace": "processing", "symbolId": "src/processing/lib/karpenter-releases:KarpenterVersion" }, "aws-dsf.processing.PySparkApplicationPackage": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "let pysparkPacker = new dsf.processing.PySparkApplicationPackage (this, 'pysparkPacker', {\n applicationName: 'my-pyspark',\n entrypointPath: '/Users/my-user/my-spark-job/app/app-pyspark.py',\n dependenciesFolder: '/Users/my-user/my-spark-job/app',\n removalPolicy: cdk.RemovalPolicy.DESTROY,\n});", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/pyspark-application-package", "stability": "stable", "summary": "A construct that takes your PySpark application, packages its virtual environment and uploads it along its entrypoint to an Amazon S3 bucket This construct requires Docker daemon installed locally to run." }, "fqn": "aws-dsf.processing.PySparkApplicationPackage", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package.ts", "line": 74 }, "parameters": [ { "docs": { "summary": "the Scope of the CDK Construct." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the ID of the CDK Construct." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "{@link PySparkApplicationPackageProps}." }, "name": "props", "type": { "fqn": "aws-dsf.processing.PySparkApplicationPackageProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package.ts", "line": 28 }, "methods": [ { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "PySparkApplicationPackage", "namespace": "processing", "properties": [ { "const": true, "docs": { "stability": "stable", "summary": "The prefix used to store artifacts on the artifact bucket." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package.ts", "line": 33 }, "name": "ARTIFACTS_PREFIX", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The S3 Bucket for storing the artifacts (entrypoint and virtual environment archive)." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package.ts", "line": 55 }, "name": "artifactsBucket", "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "docs": { "stability": "stable", "summary": "The IAM Managed Policy used by the custom resource for the assets deployment." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package.ts", "line": 67 }, "name": "assetUploadManagedPolicy", "type": { "fqn": "aws-cdk-lib.aws_iam.IManagedPolicy" } }, { "docs": { "remarks": "In case you provide your own S3 Bucket for storing the artifacts (entrypoint and virtual environment archive),\nyou must provide S3 write access to this role to upload the artifacts.", "stability": "stable", "summary": "The IAM Role used by the BucketDeployment to upload the artifacts to an s3 bucket." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package.ts", "line": 62 }, "name": "assetUploadRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "remarks": "You can pass this location to your Spark job.", "stability": "stable", "summary": "The location (generally it's an S3 URI) where the entry point is saved." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package.ts", "line": 39 }, "name": "entrypointUri", "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The Spark Config containing the configuration of virtual environment archive with all dependencies." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package.ts", "line": 50 }, "name": "sparkVenvConf", "optional": true, "type": { "primitive": "string" } }, { "docs": { "remarks": "You can pass this location to your Spark job.", "stability": "stable", "summary": "The location (generally an S3 URI) where the archive of the Python virtual environment with all dependencies is stored." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package.ts", "line": 45 }, "name": "venvArchiveUri", "optional": true, "type": { "primitive": "string" } } ], "symbolId": "src/processing/lib/spark-job/pyspark-application-package:PySparkApplicationPackage" }, "aws-dsf.processing.PySparkApplicationPackageProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for the `PySparkApplicationPackage` construct." }, "fqn": "aws-dsf.processing.PySparkApplicationPackageProps", "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package-props.ts", "line": 12 }, "name": "PySparkApplicationPackageProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "remarks": "This name is used as a parent directory in S3 to store the entrypoint and the optional virtual environment archive", "stability": "stable", "summary": "The name of the PySpark application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package-props.ts", "line": 18 }, "name": "applicationName", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "remarks": "example `~/my-project/src/entrypoint.py`", "stability": "stable", "summary": "The source path in the code base where the entrypoint is stored." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package-props.ts", "line": 24 }, "name": "entrypointPath", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- An S3 Bucket is created", "stability": "stable", "summary": "The S3 bucket where to upload the artifacts of the Spark Job This is where the entry point and archive of the virtual environment will be stored." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package-props.ts", "line": 47 }, "name": "artifactsBucket", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "abstract": true, "docs": { "default": "- A new Role would be created with least privilege permissions", "remarks": "Additional permissions would be granted to this role such as S3 Bucket permissions.", "stability": "stable", "summary": "The IAM Role used by the Lambda function." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package-props.ts", "line": 53 }, "name": "assetUploadRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "default": "- No dependencies (internal or external) are packaged. Only the entrypoint is used in the Spark Job.", "stability": "stable", "summary": "The source directory where `requirements.txt` or `pyproject.toml` file is stored. These files are used to install external AND internal Python packages. If your PySpark application has more than one Python file, you need to [package your Python project](https://packaging.python.org/en/latest/tutorials/packaging-projects/). This location must also contain a `Dockerfile` that can [create a virtual environment and build an archive](https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/using-python-libraries.html#building-python-virtual-env)." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package-props.ts", "line": 33 }, "name": "dependenciesFolder", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "Resources like Amazon cloudwatch log or Amazon S3 bucket.\nIf DESTROY is selected, the context value '@data-solutions-framework-on-aws/removeDataOnDestroy'\nin the 'cdk.json' or 'cdk.context.json' must be set to true.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package-props.ts", "line": 62 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- No virtual environment archive is packaged. Only the entrypoint can be used in the Spark Job. It is required if the `dependenciesFolder` is provided.", "remarks": "This is the output path used in the `venv-pack -o` command in your Dockerfile.", "stability": "stable", "summary": "The path of the Python virtual environment archive generated in the Docker container." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/pyspark-application-package-props.ts", "line": 40 }, "name": "venvArchivePath", "optional": true, "type": { "primitive": "string" } } ], "symbolId": "src/processing/lib/spark-job/pyspark-application-package-props:PySparkApplicationPackageProps" }, "aws-dsf.processing.SparkEmrCICDPipeline": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "custom": { "exampleMetadata": "fixture=imports-only" }, "example": "import { Bucket } from 'aws-cdk-lib/aws-s3';\n\ninterface MyApplicationStackProps extends cdk.StackProps {\n readonly stage: dsf.utils.CICDStage;\n}\n\nclass MyApplicationStack extends cdk.Stack {\n constructor(scope: cdk.Stack, props?: MyApplicationStackProps) {\n super(scope, 'MyApplicationStack');\n const bucket = new Bucket(this, 'TestBucket', {\n autoDeleteObjects: true,\n removalPolicy: cdk.RemovalPolicy.DESTROY,\n });\n new cdk.CfnOutput(this, 'BucketName', { value: bucket.bucketName });\n }\n}\n\nclass MyStackFactory implements dsf.utils.ApplicationStackFactory {\n createStack(scope: cdk.Stack, stage: dsf.utils.CICDStage): cdk.Stack {\n return new MyApplicationStack(scope, { stage });\n }\n}\n\nclass MyCICDStack extends cdk.Stack {\n constructor(scope: Construct, id: string) {\n super(scope, id);\n new dsf.processing.SparkEmrCICDPipeline(this, 'TestConstruct', {\n sparkApplicationName: 'test',\n applicationStackFactory: new MyStackFactory(),\n cdkApplicationPath: 'cdk/',\n sparkApplicationPath: 'spark/',\n sparkImage: dsf.processing.SparkImage.EMR_6_12,\n integTestScript: 'cdk/integ-test.sh',\n integTestEnv: {\n TEST_BUCKET: 'BucketName',\n },\n });\n }\n}", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-cicd-pipeline", "stability": "stable", "summary": "A CICD Pipeline to test and deploy a Spark application on Amazon EMR in cross-account environments using CDK Pipelines." }, "fqn": "aws-dsf.processing.SparkEmrCICDPipeline", "initializer": { "docs": { "stability": "stable", "summary": "Construct a new instance of the SparkCICDPipeline class." }, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts", "line": 155 }, "parameters": [ { "docs": { "summary": "the Scope of the CDK Construct." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the ID of the CDK Construct." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "the SparkCICDPipelineProps properties." }, "name": "props", "type": { "fqn": "aws-dsf.processing.SparkEmrCICDPipelineProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts", "line": 69 }, "methods": [ { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "SparkEmrCICDPipeline", "namespace": "processing", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The S3 Bucket for storing the access logs on the artifact S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts", "line": 139 }, "name": "artifactAccessLogsBucket", "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "docs": { "stability": "stable", "summary": "The S3 Bucket for storing the artifacts." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts", "line": 135 }, "name": "artifactBucket", "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "docs": { "stability": "stable", "summary": "The CodePipeline created as part of the Spark CICD Pipeline." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts", "line": 127 }, "name": "pipeline", "type": { "fqn": "aws-cdk-lib.pipelines.CodePipeline" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group for storing the CodePipeline logs." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts", "line": 143 }, "name": "pipelineLogGroup", "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The CodeCommit Repository created as part of the Spark CICD Pipeline." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts", "line": 131 }, "name": "repository", "type": { "fqn": "aws-cdk-lib.aws_codecommit.Repository" } }, { "docs": { "stability": "stable", "summary": "The CodeBuild Step for the staging stage." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts", "line": 147 }, "name": "integrationTestStage", "optional": true, "type": { "fqn": "aws-cdk-lib.pipelines.CodeBuildStep" } } ], "symbolId": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline:SparkEmrCICDPipeline" }, "aws-dsf.processing.SparkEmrCICDPipelineProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for the `SparkEmrCICDPipeline` construct." }, "fqn": "aws-dsf.processing.SparkEmrCICDPipelineProps", "kind": "interface", "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 12 }, "name": "SparkEmrCICDPipelineProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The application CDK Stack to deploy in the different CDK Pipelines Stages." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 21 }, "name": "applicationStackFactory", "type": { "fqn": "aws-dsf.utils.ApplicationStackFactory" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The name of the Spark application to be deployed." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 16 }, "name": "sparkApplicationName", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- The root of the repository", "stability": "stable", "summary": "The path to the folder that contains the CDK Application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 27 }, "name": "cdkApplicationPath", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- No environment variables", "remarks": "This is used to interact with resources created by the Application CDK Stack from within the integration tests script.\nKey is the name of the environment variable to create. Value is generally a CfnOutput name from the Application CDK Stack.", "stability": "stable", "summary": "The environment variables to create from the Application CDK Stack outputs and to pass to the integration tests." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 53 }, "name": "integTestEnv", "optional": true, "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "map" } } }, { "abstract": true, "docs": { "default": "- No permissions", "stability": "stable", "summary": "The IAM Policy statements to add permissions for running the integration tests." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 59 }, "name": "integTestPermissions", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_iam.PolicyStatement" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- No integration tests are run", "stability": "stable", "summary": "The path to the Shell script that contains integration tests." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 45 }, "name": "integTestScript", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 67 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- The root of the repository", "stability": "stable", "summary": "The path to the folder that contains the Spark Application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 33 }, "name": "sparkApplicationPath", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- [DEFAULT_SPARK_IMAGE](https://github.com/awslabs/data-solutions-framework-on-aws/blob/HEAD/framework/src/processing/lib/emr-releases.ts#L51)", "stability": "stable", "summary": "The EMR Spark image to use to run the unit tests." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props.ts", "line": 39 }, "name": "sparkImage", "optional": true, "type": { "fqn": "aws-dsf.processing.SparkImage" } } ], "symbolId": "src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline-props:SparkEmrCICDPipelineProps" }, "aws-dsf.processing.SparkEmrContainersJob": { "assembly": "aws-dsf", "base": "aws-dsf.processing.SparkJob", "docs": { "example": "import { JsonPath } from 'aws-cdk-lib/aws-stepfunctions';\n\nconst job = new dsf.processing.SparkEmrContainersJob(this, 'SparkJob', {\n jobConfig:{\n \"Name\": JsonPath.format('ge_profile-{}', JsonPath.uuid()),\n \"VirtualClusterId\": \"virtualClusterId\",\n \"ExecutionRoleArn\": \"ROLE-ARN\",\n \"JobDriver\": {\n \"SparkSubmit\": {\n \"EntryPoint\": \"s3://S3-BUCKET/pi.py\",\n \"EntryPointArguments\": [],\n \"SparkSubmitParameters\": \"--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.driver.memory=2G --conf spark.executor.cores=4\"\n },\n }\n }\n} as dsf.processing.SparkEmrContainersJobApiProps);\n\nnew cdk.CfnOutput(this, 'SparkJobStateMachine', {\n value: job.stateMachine!.stateMachineArn,\n});", "remarks": "It creates a Step Functions State Machine that orchestrates the Spark Job.", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-emr-serverless-job", "stability": "stable", "summary": "A construct to run Spark Jobs using EMR Container runtime (EMR on EKS)." }, "fqn": "aws-dsf.processing.SparkEmrContainersJob", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers.ts", "line": 47 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "union": { "types": [ { "fqn": "aws-dsf.processing.SparkEmrContainersJobProps" }, { "fqn": "aws-dsf.processing.SparkEmrContainersJobApiProps" } ] } } } ] }, "kind": "class", "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers.ts", "line": 43 }, "methods": [ { "docs": { "stability": "stable", "summary": "Grants the necessary permissions to the Step Functions StateMachine to be able to start EMR on EKS job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers.ts", "line": 164 }, "name": "grantExecutionRole", "overrides": "aws-dsf.processing.SparkJob", "parameters": [ { "docs": { "summary": "Step Functions StateMachine IAM role." }, "name": "role", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } } ], "protected": true }, { "docs": { "returns": "FailProps The error details of the failed Spark Job", "stability": "stable", "summary": "Returns the props for the Step Functions task that handles the failure if the EMR Serverless job fails." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers.ts", "line": 129 }, "name": "returnJobFailTaskProps", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions.FailProps" } } }, { "docs": { "custom": { "link": "[https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_stepfunctions_tasks.CallAwsService.html]" }, "returns": "CallAwsServiceProps", "see": "CallAwsService", "stability": "stable", "summary": "Returns the props for the Step Functions CallAwsService Construct that checks the execution status of the Spark job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers.ts", "line": 107 }, "name": "returnJobMonitorTaskProps", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions_tasks.CallAwsServiceProps" } } }, { "docs": { "custom": { "link": "[https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_stepfunctions_tasks.CallAwsService.html]" }, "remarks": "The State Machine uses [StartJobRun API](https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html).", "returns": "CallAwsServiceProps", "see": "CallAwsService", "stability": "stable", "summary": "Returns the props for the Step Functions CallAwsService Construct that starts the Spark job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers.ts", "line": 88 }, "name": "returnJobStartTaskProps", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions_tasks.CallAwsServiceProps" } } }, { "docs": { "returns": "string", "stability": "stable", "summary": "Returns the status of the EMR Serverless job that is cancelled based on the GetJobRun API response." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers.ts", "line": 156 }, "name": "returnJobStatusCancelled", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "primitive": "string" } } }, { "docs": { "returns": "string", "stability": "stable", "summary": "Returns the status of the EMR on EKS job that failed based on the GetJobRun API response." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers.ts", "line": 148 }, "name": "returnJobStatusFailed", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "primitive": "string" } } }, { "docs": { "returns": "string", "stability": "stable", "summary": "Returns the status of the EMR on EKS job that succeeded based on the GetJobRun API response." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers.ts", "line": 140 }, "name": "returnJobStatusSucceed", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "primitive": "string" } } } ], "name": "SparkEmrContainersJob", "namespace": "processing", "symbolId": "src/processing/lib/spark-job/spark-job-emr-containers:SparkEmrContainersJob" }, "aws-dsf.processing.SparkEmrContainersJobApiProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "custom": { "link": "[https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html]" }, "remarks": "Use this interface when `SparkEmrContainerJobProps` doesn't give you access to the configuration parameters you need.", "stability": "stable", "summary": "Configuration for the EMR on EKS job." }, "fqn": "aws-dsf.processing.SparkEmrContainersJobApiProps", "interfaces": [ "aws-dsf.processing.SparkJobProps" ], "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 110 }, "name": "SparkEmrContainersJobApiProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "custom": { "link": "[https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html]" }, "stability": "stable", "summary": "EMR on EKS StartJobRun API configuration." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 122 }, "name": "jobConfig", "type": { "collection": { "elementtype": { "primitive": "any" }, "kind": "map" } } }, { "abstract": true, "docs": { "default": "- 30 minutes", "stability": "stable", "summary": "Job execution timeout." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 116 }, "name": "executionTimeout", "optional": true, "type": { "fqn": "aws-cdk-lib.Duration" } } ], "symbolId": "src/processing/lib/spark-job/spark-job-emr-containers-props:SparkEmrContainersJobApiProps" }, "aws-dsf.processing.SparkEmrContainersJobProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Simplified configuration for the `SparkEmrEksJob` construct." }, "fqn": "aws-dsf.processing.SparkEmrContainersJobProps", "interfaces": [ "aws-dsf.processing.SparkJobProps" ], "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 14 }, "name": "SparkEmrContainersJobProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The IAM execution Role ARN for the EMR on EKS job." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 36 }, "name": "executionRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The Spark job name." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 19 }, "name": "name", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "see": "https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html", "stability": "stable", "summary": "The entry point for the Spark submit job run." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 41 }, "name": "sparkSubmitEntryPoint", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The EMR on EKS virtual cluster ID." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 24 }, "name": "virtualClusterId", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- No configuration is passed to the job.", "see": "https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html", "stability": "stable", "summary": "The application configuration override for the Spark submit job run." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 59 }, "name": "applicationConfiguration", "optional": true, "type": { "collection": { "elementtype": { "primitive": "any" }, "kind": "map" } } }, { "abstract": true, "docs": { "default": "- CloudWatch is not used for logging", "stability": "stable", "summary": "The CloudWatch Log Group name for log publishing." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 88 }, "name": "cloudWatchLogGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "abstract": true, "docs": { "default": "- The application name is used as the prefix", "stability": "stable", "summary": "The CloudWatch Log Group stream prefix for log publishing." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 94 }, "name": "cloudWatchLogGroupStreamPrefix", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- 30 minutes", "stability": "stable", "summary": "The execution timeout." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 65 }, "name": "executionTimeout", "optional": true, "type": { "fqn": "aws-cdk-lib.Duration" } }, { "abstract": true, "docs": { "default": "- No retry", "stability": "stable", "summary": "The maximum number of retries." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 71 }, "name": "maxRetries", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "[EMR_DEFAULT_VERSION](https://github.com/awslabs/data-solutions-framework-on-aws/blob/HEAD/framework/src/processing/lib/emr-releases.ts#L46)", "remarks": "The EMR release can be found in this [documentation](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-release-components.html)", "stability": "stable", "summary": "The EMR release version associated with the application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 31 }, "name": "releaseLabel", "optional": true, "type": { "fqn": "aws-dsf.processing.EmrRuntimeVersion" } }, { "abstract": true, "docs": { "default": "- No logging to S3", "stability": "stable", "summary": "The S3 Bucket for log publishing." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 77 }, "name": "s3LogBucket", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "abstract": true, "docs": { "default": "- No logging to S3", "stability": "stable", "summary": "The S3 Bucket prefix for log publishing." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 82 }, "name": "s3LogPrefix", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- No arguments are passed to the job.", "see": "https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html", "stability": "stable", "summary": "The arguments for the Spark submit job run." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 47 }, "name": "sparkSubmitEntryPointArguments", "optional": true, "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- No parameters are passed to the job.", "see": "https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html", "stability": "stable", "summary": "The parameters for the Spark submit job run." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 53 }, "name": "sparkSubmitParameters", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- No tags are added", "stability": "stable", "summary": "Tags to be added to the EMR Serverless job." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-containers-props.ts", "line": 100 }, "name": "tags", "optional": true, "type": { "collection": { "elementtype": { "primitive": "any" }, "kind": "map" } } } ], "symbolId": "src/processing/lib/spark-job/spark-job-emr-containers-props:SparkEmrContainersJobProps" }, "aws-dsf.processing.SparkEmrContainersRuntime": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "import { ManagedPolicy, PolicyDocument, PolicyStatement } from 'aws-cdk-lib/aws-iam';\nimport { KubectlV27Layer } from '@aws-cdk/lambda-layer-kubectl-v27';\n\nconst kubectlLayer = new KubectlV27Layer(this, 'kubectlLayer');\n\nconst emrEksCluster = dsf.processing.SparkEmrContainersRuntime.getOrCreate(this, {\n publicAccessCIDRs: ['10.0.0.0/16'],\n kubectlLambdaLayer: kubectlLayer,\n});\n\nconst virtualCluster = emrEksCluster.addEmrVirtualCluster(this, {\n name: 'example',\n createNamespace: true,\n eksNamespace: 'example',\n});\n\nconst s3Read = new PolicyDocument({\n statements: [new PolicyStatement({\n actions: [\n 's3:GetObject',\n ],\n resources: ['arn:aws:s3:::aws-data-analytics-workshop'],\n })],\n});\n\nconst s3ReadPolicy = new ManagedPolicy(this, 's3ReadPolicy', {\n document: s3Read,\n});\n\nconst execRole = emrEksCluster.createExecutionRole(this, 'ExecRole', s3ReadPolicy, 'example', 's3ReadExecRole');", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-emr-containers-runtime", "stability": "stable", "summary": "A construct to create an EKS cluster, configure it and enable it with EMR on EKS." }, "fqn": "aws-dsf.processing.SparkEmrContainersRuntime", "kind": "class", "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 86 }, "methods": [ { "docs": { "stability": "stable", "summary": "Get an existing EmrEksCluster based on the cluster name property or create a new one only one EKS cluster can exist per stack." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 154 }, "name": "getOrCreate", "parameters": [ { "docs": { "summary": "the CDK scope used to search or create the cluster." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the EmrEksClusterProps [properties]{@link EmrEksClusterProps } if created." }, "name": "props", "type": { "fqn": "aws-dsf.processing.SparkEmrContainersRuntimeProps" } } ], "returns": { "type": { "fqn": "aws-dsf.processing.SparkEmrContainersRuntime" } }, "static": true }, { "docs": { "remarks": "The method will scope the following actions `DescribeJobRun`, `TagResource` and `ListJobRuns` to the provided virtual cluster.\nIt will also scope `StartJobRun` as defined in the\n[EMR on EKS official documentation](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/iam-execution-role.html)", "stability": "stable", "summary": "A static method granting the right to start and monitor a job to an IAM Role." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 117 }, "name": "grantStartJobExecution", "parameters": [ { "docs": { "summary": "the role that will call the start job api and which needs to have the iam:PassRole permission." }, "name": "startJobRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "summary": "the role used by EMR on EKS to access resources during the job execution." }, "name": "executionRoleArn", "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "array" } } }, { "docs": { "summary": "the EMR Virtual Cluster ARN to which the job is submitted." }, "name": "virtualClusterArn", "type": { "primitive": "string" } } ], "static": true }, { "docs": { "stability": "stable", "summary": "Add a new Amazon EMR Virtual Cluster linked to Amazon EKS Cluster." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 572 }, "name": "addEmrVirtualCluster", "parameters": [ { "docs": { "summary": "of the stack where virtual cluster is deployed." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the EmrVirtualClusterProps [properties]{@link EmrVirtualClusterProps}." }, "name": "options", "type": { "fqn": "aws-dsf.processing.EmrVirtualClusterProps" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.aws_emrcontainers.CfnVirtualCluster" } } }, { "docs": { "remarks": "CfnOutput can be customized.", "stability": "stable", "summary": "Creates a new Amazon EMR managed endpoint to be used with Amazon EMR Virtual Cluster ." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 690 }, "name": "addInteractiveEndpoint", "parameters": [ { "docs": { "summary": "the scope of the stack where managed endpoint is deployed." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the CDK id for endpoint." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "the EmrManagedEndpointOptions to configure the Amazon EMR managed endpoint." }, "name": "interactiveSessionOptions", "type": { "fqn": "aws-dsf.processing.SparkEmrContainersRuntimeInteractiveSessionProps" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.CustomResource" } } }, { "docs": { "stability": "stable", "summary": "Apply the provided manifest and add the CDK dependency on EKS cluster." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 671 }, "name": "addKarpenterNodePoolAndNodeClass", "parameters": [ { "docs": { "summary": "the unique ID of the CDK resource." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "remarks": "You can use the Utils class that offers method to read yaml file and load it as a manifest", "summary": "The manifest to apply." }, "name": "manifest", "type": { "primitive": "any" } } ], "returns": { "type": { "primitive": "any" } } }, { "docs": { "remarks": "This method makes the created role assumed by the Amazon EKS cluster Open ID Connect provider.", "stability": "stable", "summary": "Create and configure a new Amazon IAM Role usable as an execution role." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 622 }, "name": "createExecutionRole", "parameters": [ { "docs": { "summary": "of the IAM role." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "of the CDK resource to be created, it should be unique across the stack." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "the execution policy to attach to the role." }, "name": "policy", "type": { "fqn": "aws-cdk-lib.aws_iam.IManagedPolicy" } }, { "docs": { "remarks": "MUST be the same as the namespace of the Virtual Cluster from which the job is submitted", "summary": "The namespace from which the role is going to be used." }, "name": "eksNamespace", "type": { "primitive": "string" } }, { "docs": { "summary": "Name to use for the role, required and is used to scope the iam role." }, "name": "name", "type": { "primitive": "string" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.aws_iam.Role" } } }, { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } }, { "docs": { "stability": "stable", "summary": "Upload podTemplates to the Amazon S3 location used by the cluster." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 651 }, "name": "uploadPodTemplate", "parameters": [ { "docs": { "summary": "the unique ID of the CDK resource." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "The local path of the yaml podTemplate files to upload." }, "name": "filePath", "type": { "primitive": "string" } } ] } ], "name": "SparkEmrContainersRuntime", "namespace": "processing", "properties": [ { "const": true, "docs": { "stability": "stable", "summary": "The default name of the EKS cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 100 }, "name": "DEFAULT_CLUSTER_NAME", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable", "summary": "The default EKS version." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 96 }, "name": "DEFAULT_EKS_VERSION", "static": true, "type": { "fqn": "aws-cdk-lib.aws_eks.KubernetesVersion" } }, { "const": true, "docs": { "stability": "stable", "summary": "The default EMR on EKS version." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 91 }, "name": "DEFAULT_EMR_EKS_VERSION", "static": true, "type": { "fqn": "aws-dsf.processing.EmrContainersRuntimeVersion" } }, { "const": true, "docs": { "stability": "stable", "summary": "The default CIDR when the VPC is created." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 104 }, "name": "DEFAULT_VPC_CIDR", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The IAM role used by the tooling managed nodegroup hosting core Kubernetes controllers like EBS CSI driver, core dns." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 178 }, "name": "ec2InstanceNodeGroupRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The EKS cluster created by the construct if it is not provided." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 169 }, "name": "eksCluster", "type": { "fqn": "aws-cdk-lib.aws_eks.Cluster" } }, { "docs": { "stability": "stable", "summary": "The VPC used by the EKS cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 247 }, "name": "vpc", "type": { "fqn": "aws-cdk-lib.aws_ec2.IVpc" } }, { "docs": { "stability": "stable", "summary": "The bucket holding podtemplates referenced in the configuration override for the job." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 210 }, "name": "assetBucket", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "docs": { "stability": "stable", "summary": "The IAM role used to upload assets (pod templates) on S3." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 276 }, "name": "assetUploadBucketRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The IAM Role used by IRSA for the aws-node daemonset." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 173 }, "name": "awsNodeRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The configuration override for the spark application to use with the default nodes for criticale jobs." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 202 }, "name": "criticalDefaultConfig", "optional": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The IAM Role created for the EBS CSI controller." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 239 }, "name": "csiDriverIrsaRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The KMS key used for storing EKS secrets." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 272 }, "name": "eksSecretKmsKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "docs": { "stability": "stable", "summary": "The Service Linked role created for EMR." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 268 }, "name": "emrServiceRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.CfnServiceLinkedRole" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group for the VPC flow log when the VPC is created." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 263 }, "name": "flowLogGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The KMS Key used for the VPC flow logs when the VPC is created." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 251 }, "name": "flowLogKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "docs": { "stability": "stable", "summary": "The IAM Role used for the VPC flow logs when the VPC is created." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 255 }, "name": "flowLogRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The rules used by Karpenter to track node health, rules are defined in the cloudformation below https://raw.githubusercontent.com/aws/karpenter/\"${KARPENTER_VERSION}\"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 193 }, "name": "karpenterEventRules", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_events.IRule" }, "kind": "array" } } }, { "docs": { "stability": "stable", "summary": "The IAM role created for the Karpenter controller." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 243 }, "name": "karpenterIrsaRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The SQS queue used by Karpenter to receive critical events from AWS services which may affect your nodes." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 182 }, "name": "karpenterQueue", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_sqs.IQueue" } }, { "docs": { "stability": "stable", "summary": "The security group used by the EC2NodeClass of the default nodes." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 186 }, "name": "karpenterSecurityGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.ISecurityGroup" } }, { "docs": { "stability": "stable", "summary": "The configuration override for the spark application to use with the default nodes dedicated for notebooks." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 198 }, "name": "notebookDefaultConfig", "optional": true, "type": { "primitive": "any" } }, { "docs": { "stability": "stable", "summary": "The S3 location holding the driver pod tempalte for critical nodes." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 215 }, "name": "podTemplateS3LocationCriticalDriver", "optional": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The S3 location holding the executor pod tempalte for critical nodes." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 219 }, "name": "podTemplateS3LocationCriticalExecutor", "optional": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The S3 location holding the driver pod tempalte for shared nodes." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 223 }, "name": "podTemplateS3LocationDriverShared", "optional": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The S3 location holding the executor pod tempalte for shared nodes." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 227 }, "name": "podTemplateS3LocationExecutorShared", "optional": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The S3 location holding the driver pod tempalte for interactive sessions." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 231 }, "name": "podTemplateS3LocationNotebookDriver", "optional": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The S3 location holding the executor pod tempalte for interactive sessions." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 235 }, "name": "podTemplateS3LocationNotebookExecutor", "optional": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The S3 VPC endpoint attached to the private subnets of the VPC when VPC is created." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 259 }, "name": "s3VpcEndpoint", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.IGatewayVpcEndpoint" } }, { "docs": { "stability": "stable", "summary": "The configuration override for the spark application to use with the default nodes for none criticale jobs." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts", "line": 206 }, "name": "sharedDefaultConfig", "optional": true, "type": { "primitive": "string" } } ], "symbolId": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime:SparkEmrContainersRuntime" }, "aws-dsf.processing.SparkEmrContainersRuntimeInteractiveSessionProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "The properties for the EMR Managed Endpoint to create." }, "fqn": "aws-dsf.processing.SparkEmrContainersRuntimeInteractiveSessionProps", "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-interactive-endpoint-props.ts", "line": 10 }, "name": "SparkEmrContainersRuntimeInteractiveSessionProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The Amazon IAM role used as the execution role, this role must provide access to all the AWS resource a user will interact with These can be S3, DynamoDB, Glue Catalog." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-interactive-endpoint-props.ts", "line": 23 }, "name": "executionRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The name of the EMR managed endpoint." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-interactive-endpoint-props.ts", "line": 14 }, "name": "managedEndpointName", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The Id of the Amazon EMR virtual cluster containing the managed endpoint." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-interactive-endpoint-props.ts", "line": 18 }, "name": "virtualClusterId", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- Configuration related to the [default nodegroup for notebook]{@link EmrEksNodegroup.NOTEBOOK_EXECUTOR }", "stability": "stable", "summary": "The JSON configuration overrides for Amazon EMR on EKS configuration attached to the managed endpoint." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-interactive-endpoint-props.ts", "line": 33 }, "name": "configurationOverrides", "optional": true, "type": { "primitive": "any" } }, { "abstract": true, "docs": { "default": "- The [default Amazon EMR version]{@link EmrEksCluster.DEFAULT_EMR_VERSION }", "stability": "stable", "summary": "The Amazon EMR version to use." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-interactive-endpoint-props.ts", "line": 28 }, "name": "emrOnEksVersion", "optional": true, "type": { "fqn": "aws-dsf.processing.EmrContainersRuntimeVersion" } } ], "symbolId": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-interactive-endpoint-props:SparkEmrContainersRuntimeInteractiveSessionProps" }, "aws-dsf.processing.SparkEmrContainersRuntimeProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "The properties for the `SparkEmrContainerRuntime` Construct class." }, "fqn": "aws-dsf.processing.SparkEmrContainersRuntimeProps", "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 18 }, "name": "SparkEmrContainersRuntimeProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "remarks": "Starting k8s 1.22, CDK no longer bundle the Kubectl layer with the code due to breaking npm package size.\nA layer needs to be passed to the Construct.\n\nThe CDK [documentation](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_eks.KubernetesVersion.html#static-v1_22)\ncontains the libraries that you should add for the right Kubernetes version.", "stability": "stable", "summary": "The Lambda Layer with Kubectl to use for EKS Cluster setup." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 63 }, "name": "kubectlLambdaLayer", "type": { "fqn": "aws-cdk-lib.aws_lambda.ILayerVersion" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The CIDR blocks that are allowed to access to your cluster’s public Kubernetes API server endpoint." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 86 }, "name": "publicAccessCIDRs", "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- true", "stability": "stable", "summary": "Flag to create an IAM Service Linked Role for EMR on EKS." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 97 }, "name": "createEmrOnEksServiceLinkedRole", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- true", "stability": "stable", "summary": "The flag to create default Karpenter Node Provisioners for: * Critical jobs which use on-demand instances, high speed disks and workload isolation * Shared workloads which use EC2 Spot Instances and no isolation to optimize costs * Notebooks which leverage a cost optimized configuration for running EMR managed endpoints and spark drivers/executors." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 49 }, "name": "defaultNodes", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- A role is created with AmazonEKSWorkerNodePolicy, AmazonEC2ContainerRegistryReadOnly,\nAmazonSSMManagedInstanceCore and AmazonEKS_CNI_Policy AWS managed policies.", "stability": "stable", "summary": "The IAM Role used for the cluster nodes instance profile." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 92 }, "name": "ec2InstanceRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "default": "- An admin role must be passed if `eksCluster` property is not set.", "remarks": "It will give access to kubernetes cluster from the AWS console.\nYou will use this role to manage the EKS cluster and grant other access to it.", "stability": "stable", "summary": "The IAM Role to configure in the EKS master roles." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 29 }, "name": "eksAdminRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "default": "- An EKS Cluster is created", "remarks": "The cluster needs to be created in the same CDK Stack.\nIf the EKS Cluster is provided, the cluster AddOns and all the controllers (ALB Ingress controller, Karpenter...) need to be configured.\nWhen providing an EKS cluster, the methods for adding nodegroups can still be used. They implement the best practices for running Spark on EKS.", "stability": "stable", "summary": "The EKS Cluster to setup EMR on." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 36 }, "name": "eksCluster", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_eks.Cluster" } }, { "abstract": true, "docs": { "default": "- The [DEFAULT_CLUSTER_NAME](https://github.com/awslabs/data-solutions-framework-on-aws/blob/HEAD/framework/src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts#L65)", "stability": "stable", "summary": "The name of the EKS cluster to create." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 23 }, "name": "eksClusterName", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- A new VPC is created.", "remarks": "VPC should have at least two private and public subnets in different Availability Zones.\nAll private subnets must have the following tags:\n * 'for-use-with-amazon-emr-managed-policies'='true'\n * 'kubernetes.io/role/internal-elb'='1'\nAll public subnets must have the following tag:\n * 'kubernetes.io/role/elb'='1'\nCannot be combined with `vpcCidr`. If combined, `vpcCidr` takes precedence and a new VPC is created.", "stability": "stable", "summary": "The VPC to use when creating the EKS cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 82 }, "name": "eksVpc", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.IVpc" } }, { "abstract": true, "docs": { "default": "- [DEFAULT_KARPENTER_VERSION](https://github.com/awslabs/data-solutions-framework-on-aws/blob/HEAD/framework/src/processing/lib/karpenter-releases.ts#L11)", "stability": "stable", "summary": "The Karpenter version to use for autoscaling nodes in the EKS Cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 54 }, "name": "karpenterVersion", "optional": true, "type": { "fqn": "aws-dsf.processing.KarpenterVersion" } }, { "abstract": true, "docs": { "default": "- [DEFAULT_EKS_VERSION](https://github.com/awslabs/data-solutions-framework-on-aws/blob/HEAD/framework/src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts#L61)", "stability": "stable", "summary": "The Kubernetes version used to create the EKS Cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 41 }, "name": "kubernetesVersion", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_eks.KubernetesVersion" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 105 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- none", "stability": "stable", "summary": "The tags assigned to the EKS cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 112 }, "name": "tags", "optional": true, "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "map" } } }, { "abstract": true, "docs": { "default": "- The CIDR 10.0.0.0/16 is used", "remarks": "If provided, a VPC with three public subnets and three private subnets is created.\nThe size of the private subnets is four time the one of the public subnet.", "stability": "stable", "summary": "The CIDR of the VPC to use when creating the EKS cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props.ts", "line": 70 }, "name": "vpcCidr", "optional": true, "type": { "primitive": "string" } } ], "symbolId": "src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime-props:SparkEmrContainersRuntimeProps" }, "aws-dsf.processing.SparkEmrServerlessJob": { "assembly": "aws-dsf", "base": "aws-dsf.processing.SparkJob", "docs": { "example": "import { PolicyDocument, PolicyStatement } from 'aws-cdk-lib/aws-iam';\nimport { JsonPath } from 'aws-cdk-lib/aws-stepfunctions';\n\nconst myExecutionRole = dsf.processing.SparkEmrServerlessRuntime.createExecutionRole(this, 'execRole1');\nconst job = new dsf.processing.SparkEmrServerlessJob(this, 'SparkJob', {\n jobConfig:{\n \"Name\": JsonPath.format('ge_profile-{}', JsonPath.uuid()),\n \"ApplicationId\": \"APPLICATION_ID\",\n \"ExecutionRoleArn\": myExecutionRole.roleArn,\n \"JobDriver\": {\n \"SparkSubmit\": {\n \"EntryPoint\": \"s3://S3-BUCKET/pi.py\",\n \"EntryPointArguments\": [],\n \"SparkSubmitParameters\": \"--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.driver.memory=2G --conf spark.executor.cores=4\"\n },\n }\n }\n} as dsf.processing.SparkEmrServerlessJobApiProps);", "remarks": "Creates a State Machine that orchestrates the Spark Job.", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-emr-serverless-job", "stability": "stable", "summary": "A construct to run Spark Jobs using EMR Serverless." }, "fqn": "aws-dsf.processing.SparkEmrServerlessJob", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 49 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "union": { "types": [ { "fqn": "aws-dsf.processing.SparkEmrServerlessJobProps" }, { "fqn": "aws-dsf.processing.SparkEmrServerlessJobApiProps" } ] } } } ] }, "kind": "class", "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 41 }, "methods": [ { "docs": { "see": "SparkRuntimeServerless.grantJobExecution", "stability": "stable", "summary": "Grants the necessary permissions to the Step Functions StateMachine to be able to start EMR Serverless job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 169 }, "name": "grantExecutionRole", "overrides": "aws-dsf.processing.SparkJob", "parameters": [ { "docs": { "summary": "Step Functions StateMachine IAM role." }, "name": "role", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } } ], "protected": true }, { "docs": { "returns": "FailProps The error details of the failed Spark Job", "stability": "stable", "summary": "Returns the props for the step function task that handles the failure if the EMR Serverless job fails." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 131 }, "name": "returnJobFailTaskProps", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions.FailProps" } } }, { "docs": { "custom": { "link": "[https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_stepfunctions_tasks.CallAwsService.html]" }, "returns": "CallAwsServiceProps", "see": "CallAwsService", "stability": "stable", "summary": "Returns the props for the Step Functions CallAwsService Construct that checks the execution status of the Spark job, it calls the [GetJobRun API](https://docs.aws.amazon.com/emr-serverless/latest/APIReference/API_GetJobRun.html)." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 109 }, "name": "returnJobMonitorTaskProps", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions_tasks.CallAwsServiceProps" } } }, { "docs": { "custom": { "link": "[https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_stepfunctions_tasks.CallAwsService.html]" }, "returns": "CallAwsServiceProps", "see": "CallAwsService", "stability": "stable", "summary": "Returns the props for the Step Functions CallAwsService Construct that starts the Spark job, it calls the [StartJobRun API](https://docs.aws.amazon.com/emr-serverless/latest/APIReference/API_StartJobRun.html)." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 91 }, "name": "returnJobStartTaskProps", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions_tasks.CallAwsServiceProps" } } }, { "docs": { "returns": "string", "stability": "stable", "summary": "Returns the status of the EMR Serverless job that is cancelled based on the GetJobRun API response." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 159 }, "name": "returnJobStatusCancelled", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "primitive": "string" } } }, { "docs": { "returns": "string", "stability": "stable", "summary": "Returns the status of the EMR Serverless job that failed based on the GetJobRun API response." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 151 }, "name": "returnJobStatusFailed", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "primitive": "string" } } }, { "docs": { "returns": "string", "stability": "stable", "summary": "Returns the status of the EMR Serverless job that succeeded based on the GetJobRun API response." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 143 }, "name": "returnJobStatusSucceed", "overrides": "aws-dsf.processing.SparkJob", "protected": true, "returns": { "type": { "primitive": "string" } } } ], "name": "SparkEmrServerlessJob", "namespace": "processing", "properties": [ { "docs": { "remarks": "Use this property to add additional IAM permissions if necessary.", "stability": "stable", "summary": "The Spark job execution role." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless.ts", "line": 46 }, "name": "sparkJobExecutionRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } } ], "symbolId": "src/processing/lib/spark-job/spark-job-emr-serverless:SparkEmrServerlessJob" }, "aws-dsf.processing.SparkEmrServerlessJobApiProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "custom": { "link": "[https://docs.aws.amazon.com/emr-serverless/latest/APIReference/API_StartJobRun.html]" }, "remarks": "Use this interface when `SparkEmrServerlessJobProps` doesn't give you access to the configuration parameters you need.", "stability": "stable", "summary": "Configuration for the EMR Serverless Job API." }, "fqn": "aws-dsf.processing.SparkEmrServerlessJobApiProps", "interfaces": [ "aws-dsf.processing.SparkJobProps" ], "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 125 }, "name": "SparkEmrServerlessJobApiProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "custom": { "link": "[https://docs.aws.amazon.com/emr-serverless/latest/APIReference/API_StartJobRun.html]" }, "stability": "stable", "summary": "EMR Serverless Job Configuration." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 131 }, "name": "jobConfig", "type": { "collection": { "elementtype": { "primitive": "any" }, "kind": "map" } } } ], "symbolId": "src/processing/lib/spark-job/spark-job-emr-serverless-props:SparkEmrServerlessJobApiProps" }, "aws-dsf.processing.SparkEmrServerlessJobProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable" }, "fqn": "aws-dsf.processing.SparkEmrServerlessJobProps", "interfaces": [ "aws-dsf.processing.SparkJobProps" ], "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 14 }, "name": "SparkEmrServerlessJobProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The EMR Serverless Application to execute the Spark Job." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 24 }, "name": "applicationId", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The IAM execution Role for the EMR Serverless job." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 29 }, "name": "executionRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The Spark Job name." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 19 }, "name": "name", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "see": "https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html", "stability": "stable", "summary": "The entry point for the Spark submit job run." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 34 }, "name": "sparkSubmitEntryPoint", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- No configuration is passed to the job.", "see": "https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html", "stability": "stable", "summary": "The application configuration override for the Spark submit job run." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 52 }, "name": "applicationConfiguration", "optional": true, "type": { "collection": { "elementtype": { "primitive": "any" }, "kind": "map" } } }, { "abstract": true, "docs": { "default": "- No encryption", "stability": "stable", "summary": "The KMS Key for encrypting logs on CloudWatch." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 99 }, "name": "cloudWatchEncryptionKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "default": "- No logging to CloudWatch", "stability": "stable", "summary": "The CloudWatch Log Group name for log publishing." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 93 }, "name": "cloudWatchLogGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "abstract": true, "docs": { "default": "- No prefix is used", "stability": "stable", "summary": "The CloudWatch Log Group Stream prefix for log publishing." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 105 }, "name": "cloudWatchLogGroupStreamPrefix", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The types of logs to log in CloudWatch Log." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 110 }, "name": "cloudWatchLogtypes", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- 30 minutes", "stability": "stable", "summary": "The execution timeout." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 58 }, "name": "executionTimeout", "optional": true, "type": { "fqn": "aws-cdk-lib.Duration" } }, { "abstract": true, "docs": { "default": "- true", "stability": "stable", "summary": "Enable Spark persistent UI logs in EMR managed storage." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 64 }, "name": "persistentAppUi", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- Use EMR managed Key", "stability": "stable", "summary": "The KMS Key ARN to encrypt Spark persistent UI logs in EMR managed storage." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 70 }, "name": "persistentAppUIKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "default": "- No logging to S3", "stability": "stable", "summary": "The S3 Bucket for log publishing." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 76 }, "name": "s3LogBucket", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "abstract": true, "docs": { "default": "- No encryption", "stability": "stable", "summary": "The KMS Key for encrypting logs on S3." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 87 }, "name": "s3LogEncryptionKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "default": "- No logging to S3", "stability": "stable", "summary": "The S3 Bucket prefix for log publishing." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 81 }, "name": "s3LogPrefix", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- No arguments are passed to the job.", "see": "https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html", "stability": "stable", "summary": "The arguments for the Spark submit job run." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 40 }, "name": "sparkSubmitEntryPointArguments", "optional": true, "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- No parameters are passed to the job.", "see": "https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html", "stability": "stable", "summary": "The parameters for the Spark submit job run." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 46 }, "name": "sparkSubmitParameters", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- No tags are added", "stability": "stable", "summary": "Tags to be added to the EMR Serverless job." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-emr-serverless-props.ts", "line": 116 }, "name": "tags", "optional": true, "type": { "collection": { "elementtype": { "primitive": "any" }, "kind": "map" } } } ], "symbolId": "src/processing/lib/spark-job/spark-job-emr-serverless-props:SparkEmrServerlessJobProps" }, "aws-dsf.processing.SparkEmrServerlessRuntime": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "import { Role, AccountRootPrincipal } from 'aws-cdk-lib/aws-iam';\n\nconst serverlessRuntime = new dsf.processing.SparkEmrServerlessRuntime(this, 'EmrApp', {\n name: 'SparkRuntimeServerless',\n});\n\nconst executionRole = dsf.processing.SparkEmrServerlessRuntime.createExecutionRole(this, 'ExecutionRole')\n\nconst submitterRole = new Role (this, 'SubmitterRole', {\n assumedBy: new AccountRootPrincipal(),\n});\n\ndsf.processing.SparkEmrServerlessRuntime.grantStartJobExecution(submitterRole, [executionRole.roleArn], ['EMR-serverless-app-ID']);", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-emr-serverless-runtime", "stability": "stable", "summary": "A construct to create a Spark EMR Serverless Application, along with methods to create IAM roles having the least privilege." }, "fqn": "aws-dsf.processing.SparkEmrServerlessRuntime", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 180 }, "parameters": [ { "docs": { "summary": "the Scope of the CDK Construct." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the ID of the CDK Construct." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "{@link SparkEmrServerlessRuntimeProps}." }, "name": "props", "type": { "fqn": "aws-dsf.processing.SparkEmrServerlessRuntimeProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 34 }, "methods": [ { "docs": { "remarks": "If no `executionRolePolicyDocument` or `iamPolicyName`\nThe method will return a role with only a trust policy to EMR Servereless service principal.\nYou can use this role then to grant access to any resources you control.", "stability": "stable", "summary": "A static method creating an execution IAM role that can be assumed by EMR Serverless The method returns the role it creates." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 48 }, "name": "createExecutionRole", "parameters": [ { "docs": { "summary": "the scope in which to create the role." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "passed to the IAM Role construct object." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "remarks": "These are IAM policies needed by the job.\nThis parameter is mutually execlusive with iamPolicyName.", "summary": "the inline policy document to attach to the role." }, "name": "executionRolePolicyDocument", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.PolicyDocument" } }, { "docs": { "summary": "the IAM policy name to attach to the role, this is mutually execlusive with executionRolePolicyDocument." }, "name": "iamPolicyName", "optional": true, "type": { "primitive": "string" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, "static": true }, { "docs": { "remarks": "The method will also attach an iam:PassRole permission limited to the IAM Job Execution roles passed", "stability": "stable", "summary": "A static method granting the right to start and monitor a job to an IAM Role." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 83 }, "name": "grantStartJobExecution", "parameters": [ { "docs": { "summary": "the role that will call the start job api and which needs to have the iam:PassRole permission." }, "name": "startJobRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "summary": "the role used by EMR Serverless to access resources during the job execution." }, "name": "executionRoleArn", "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "array" } } }, { "docs": { "summary": "the EMR Serverless aplication ARN, this is used by the method to limit the EMR Serverless applications the role can submit job to." }, "name": "applicationArns", "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "array" } } } ], "static": true }, { "docs": { "remarks": "The method will also attach an iam:PassRole permission to limited to the IAM Job Execution roles passed.\nThe excution role will be able to submit job to the EMR Serverless application created by the construct.", "stability": "stable", "summary": "A method which will grant an IAM Role the right to start and monitor a job." }, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 246 }, "name": "grantStartExecution", "parameters": [ { "docs": { "summary": "the role that will call the start job api and which need to have the iam:PassRole permission." }, "name": "startJobRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "summary": "the role use by EMR Serverless to access resources during the job execution." }, "name": "executionRoleArn", "type": { "primitive": "string" } } ] }, { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "SparkEmrServerlessRuntime", "namespace": "processing", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The EMR Serverless application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 145 }, "name": "application", "type": { "fqn": "aws-cdk-lib.aws_emrserverless.CfnApplication" } }, { "docs": { "stability": "stable", "summary": "If no VPC is provided, one is created by default along with a security group attached to the EMR Serverless Application This attribute is used to expose the security group, if you provide your own security group through the {@link SparkEmrServerlessRuntimeProps} the attribute will be `undefined`." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 152 }, "name": "emrApplicationSecurityGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.ISecurityGroup" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group for the VPC flow log when the VPC is created." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 169 }, "name": "flowLogGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The KMS Key used for the VPC flow log when the VPC is created." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 161 }, "name": "flowLogKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "docs": { "stability": "stable", "summary": "The IAM Role used for the VPC flow log when the VPC is created." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 165 }, "name": "flowLogRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 172 }, "name": "s3VpcEndpoint", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.IGatewayVpcEndpoint" } }, { "docs": { "stability": "stable", "summary": "The VPC used by the EKS cluster." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless.ts", "line": 157 }, "name": "vpc", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.IVpc" } } ], "symbolId": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless:SparkEmrServerlessRuntime" }, "aws-dsf.processing.SparkEmrServerlessRuntimeProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for the `SparkEmrServerlessRuntime` construct." }, "fqn": "aws-dsf.processing.SparkEmrServerlessRuntimeProps", "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 11 }, "name": "SparkEmrServerlessRuntimeProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "remarks": "The name must be less than 64 characters.\n*Pattern* : `^[A-Za-z0-9._\\\\/#-]+$`", "stability": "stable", "summary": "The name of the application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 40 }, "name": "name", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- x86_64", "stability": "stable", "summary": "The CPU architecture type of the application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 80 }, "name": "architecture", "optional": true, "type": { "fqn": "aws-dsf.utils.Architecture" } }, { "abstract": true, "docs": { "default": "- True", "stability": "stable", "summary": "The configuration for an application to automatically start on job submission." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 74 }, "name": "autoStartConfiguration", "optional": true, "type": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "fqn": "aws-cdk-lib.aws_emrserverless.CfnApplication.AutoStartConfigurationProperty" } ] } } }, { "abstract": true, "docs": { "default": "- The application is stopped after 15 minutes of idle time", "stability": "stable", "summary": "The configuration for an application to automatically stop after a certain amount of time being idle." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 69 }, "name": "autoStopConfiguration", "optional": true, "type": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "fqn": "aws-cdk-lib.aws_emrserverless.CfnApplication.AutoStopConfigurationProperty" } ] } } }, { "abstract": true, "docs": { "default": "- EMR base image is used for both the Spark Driver and the Spark Executor", "see": "https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/application-custom-image.html", "stability": "stable", "summary": "The unique custom image configuration used for both the Spark Driver and the Spark Executor." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 58 }, "name": "imageConfiguration", "optional": true, "type": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "fqn": "aws-cdk-lib.aws_emrserverless.CfnApplication.ImageConfigurationInputProperty" } ] } } }, { "abstract": true, "docs": { "default": "- No pre-initialized capacity is used", "see": "https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html", "stability": "stable", "summary": "The pre-initialized capacity of the application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 52 }, "name": "initialCapacity", "optional": true, "type": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "collection": { "elementtype": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "fqn": "aws-cdk-lib.aws_emrserverless.CfnApplication.InitialCapacityConfigKeyValuePairProperty" } ] } }, "kind": "array" } } ] } } }, { "abstract": true, "docs": { "default": "- Depending on the EMR version", "remarks": "This is cumulative across all workers at any given point in time during the lifespan of the application is created. No new resources will be created once any one of the defined limits is hit.", "stability": "stable", "summary": "The maximum capacity of the application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 46 }, "name": "maximumCapacity", "optional": true, "type": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "fqn": "aws-cdk-lib.aws_emrserverless.CfnApplication.MaximumAllowedResourcesProperty" } ] } } }, { "abstract": true, "docs": { "default": "- a VPC and a security group are created, these are accessed as construct attribute.", "remarks": "If no configuration is created, the a VPC with 3 public subnets and 3 private subnets is created\nThe 3 public subnets and 3 private subnets are each created in an Availability Zone (AZ)\nThe VPC has one NAT Gateway per AZ and an S3 endpoint", "stability": "stable", "summary": "The network configuration for customer VPC connectivity for the application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 28 }, "name": "networkConfiguration", "optional": true, "type": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "fqn": "aws-cdk-lib.aws_emrserverless.CfnApplication.NetworkConfigurationProperty" } ] } } }, { "abstract": true, "docs": { "default": "[EMR_DEFAULT_VERSION](https://github.com/awslabs/data-solutions-framework-on-aws/blob/HEAD/framework/src/processing/lib/emr-releases.ts#L46)", "remarks": "The EMR release can be found in this [documentation](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-release-components.html)", "stability": "stable", "summary": "The EMR release version associated with the application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 86 }, "name": "releaseLabel", "optional": true, "type": { "fqn": "aws-dsf.processing.EmrRuntimeVersion" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 35 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- No custom configuration is used", "see": "https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/default-configs.html", "stability": "stable", "summary": "The runtime and monitoring configurations to used as defaults for all of the job runs of this application." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 64 }, "name": "runtimeConfiguration", "optional": true, "type": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_emrserverless.CfnApplication.ConfigurationObjectProperty" }, "kind": "array" } } ] } } }, { "abstract": true, "docs": { "default": "- EMR base image is used for both the Spark Driver and the Spark Executor", "see": "https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/application-custom-image.html", "stability": "stable", "summary": "The different custom image configurations used for the Spark Driver and the Spark Executor." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts", "line": 17 }, "name": "workerTypeSpecifications", "optional": true, "type": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "collection": { "elementtype": { "union": { "types": [ { "fqn": "aws-cdk-lib.IResolvable" }, { "fqn": "aws-cdk-lib.aws_emrserverless.CfnApplication.WorkerTypeSpecificationInputProperty" } ] } }, "kind": "map" } } ] } } } ], "symbolId": "src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props:SparkEmrServerlessRuntimeProps" }, "aws-dsf.processing.SparkImage": { "assembly": "aws-dsf", "docs": { "stability": "stable", "summary": "The list of supported Spark images to use in the SparkCICDPipeline." }, "fqn": "aws-dsf.processing.SparkImage", "kind": "enum", "locationInModule": { "filename": "src/processing/lib/emr-releases.ts", "line": 57 }, "members": [ { "docs": { "stability": "stable" }, "name": "EMR_7_0" }, { "docs": { "stability": "stable" }, "name": "EMR_6_15" }, { "docs": { "stability": "stable" }, "name": "EMR_6_14" }, { "docs": { "stability": "stable" }, "name": "EMR_6_13" }, { "docs": { "stability": "stable" }, "name": "EMR_6_12" }, { "docs": { "stability": "stable" }, "name": "EMR_6_11" }, { "docs": { "stability": "stable" }, "name": "EMR_6_10" }, { "docs": { "stability": "stable" }, "name": "EMR_6_9" } ], "name": "SparkImage", "namespace": "processing", "symbolId": "src/processing/lib/emr-releases:SparkImage" }, "aws-dsf.processing.SparkJob": { "abstract": true, "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "remarks": "Creates an AWS Step Functions State Machine that orchestrates the Spark Job.", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-emr-serverless-job\n\nAvailable implementations:\n* {@link SparkEmrServerlessJob } for Emr Serverless implementation\n* {@link SparkEmrEksJob } for EMR On EKS implementation", "stability": "stable", "summary": "A base construct to run Spark Jobs." }, "fqn": "aws-dsf.processing.SparkJob", "initializer": { "docs": { "stability": "stable", "summary": "Constructs a new instance of the SparkJob class." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 62 }, "parameters": [ { "docs": { "summary": "the Scope of the CDK Construct." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the ID of the CDK Construct." }, "name": "id", "type": { "primitive": "string" } }, { "name": "trackingTag", "type": { "primitive": "string" } }, { "docs": { "summary": "the SparkJobProps properties." }, "name": "props", "type": { "fqn": "aws-dsf.processing.SparkJobProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 28 }, "methods": [ { "docs": { "default": "- Server-side encryption managed by CloudWatch Logs.", "returns": "LogGroup CloudWatch Logs group.", "stability": "stable", "summary": "Creates an encrypted CloudWatch Logs group to store the Spark job logs." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 209 }, "name": "createCloudWatchLogsLogGroup", "parameters": [ { "docs": { "summary": "CloudWatch Logs group name of cloudwatch log group to store the Spark job logs." }, "name": "name", "type": { "primitive": "string" } }, { "docs": { "summary": "KMS Key ARN for encryption." }, "name": "encryptionKeyArn", "optional": true, "type": { "primitive": "string" } } ], "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } } }, { "docs": { "default": "- Master KMS key of the account", "remarks": "The bucket is created with SSE encryption (KMS managed or provided by user).", "returns": "string S3 path to store the logs.", "stability": "stable", "summary": "Creates or import an S3 bucket to store the logs of the Spark job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 187 }, "name": "createS3LogBucket", "parameters": [ { "docs": { "summary": "The S3 Bucket to store the logs of the Spark job." }, "name": "s3LogBucket", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "docs": { "summary": "The prefix to store logs in the Log Bucket." }, "name": "s3LogPrefix", "optional": true, "type": { "primitive": "string" } }, { "docs": { "summary": "The KMS Key for encryption." }, "name": "encryptionKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } } ], "protected": true, "returns": { "type": { "primitive": "string" } } }, { "docs": { "custom": { "defautl": "30 minutes" }, "default": "no schedule", "remarks": "This is a default implementation that can be overridden by the extending class.", "returns": "StateMachine", "stability": "stable", "summary": "Creates a State Machine that orchestrates the Spark Job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 126 }, "name": "createStateMachine", "parameters": [ { "docs": { "summary": "Timeout for the state machine." }, "name": "jobTimeout", "type": { "fqn": "aws-cdk-lib.Duration" } }, { "docs": { "summary": "Schedule to run the state machine." }, "name": "schedule", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_events.Schedule" } } ], "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions.StateMachine" } } }, { "abstract": true, "docs": { "stability": "stable", "summary": "Grants the execution role to the Step Functions state machine." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 118 }, "name": "grantExecutionRole", "parameters": [ { "name": "role", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } } ], "protected": true }, { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } }, { "abstract": true, "docs": { "returns": "FailProps", "stability": "stable", "summary": "Parameters for Step Functions task that fails the Spark job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 87 }, "name": "returnJobFailTaskProps", "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions.FailProps" } } }, { "abstract": true, "docs": { "returns": "CallAwsServiceProps", "stability": "stable", "summary": "Parameters for Step Functions task that monitors the Spark job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 81 }, "name": "returnJobMonitorTaskProps", "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions_tasks.CallAwsServiceProps" } } }, { "abstract": true, "docs": { "returns": "CallAwsServiceProps", "stability": "stable", "summary": "Parameters for Step Functions task that runs the Spark job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 75 }, "name": "returnJobStartTaskProps", "protected": true, "returns": { "type": { "fqn": "aws-cdk-lib.aws_stepfunctions_tasks.CallAwsServiceProps" } } }, { "abstract": true, "docs": { "stability": "stable", "summary": "Returns the status of the Spark job that is cancelled based on the GetJobRun API response." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 104 }, "name": "returnJobStatusCancelled", "protected": true, "returns": { "type": { "primitive": "string" } } }, { "abstract": true, "docs": { "returns": "string", "stability": "stable", "summary": "Returns the status of the Spark job that failed based on the GetJobRun API response." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 99 }, "name": "returnJobStatusFailed", "protected": true, "returns": { "type": { "primitive": "string" } } }, { "abstract": true, "docs": { "returns": "string", "stability": "stable", "summary": "Returns the status of the Spark job that succeeded based on the GetJobRun API response." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 93 }, "name": "returnJobStatusSucceed", "protected": true, "returns": { "type": { "primitive": "string" } } } ], "name": "SparkJob", "namespace": "processing", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The loudWatch Logs Group for the Spark job logs." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 48 }, "name": "emrJobLogGroup", "optional": true, "protected": true, "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The S3 Bucket for the Spark job logs." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 43 }, "name": "s3LogBucket", "optional": true, "protected": true, "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "docs": { "stability": "stable", "summary": "The Step Functions State Machine created to orchestrate the Spark Job." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 33 }, "name": "stateMachine", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_stepfunctions.StateMachine" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group used by the State Machine." }, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job.ts", "line": 38 }, "name": "stateMachineLogGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } } ], "symbolId": "src/processing/lib/spark-job/spark-job:SparkJob" }, "aws-dsf.processing.SparkJobProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "The properties for the `SparkJob` construct." }, "fqn": "aws-dsf.processing.SparkJobProps", "kind": "interface", "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-props.ts", "line": 10 }, "name": "SparkJobProps", "namespace": "processing", "properties": [ { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-props.ts", "line": 18 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- The Step Functions State Machine is not scheduled.", "stability": "stable", "summary": "The Schedule to run the Step Functions state machine." }, "immutable": true, "locationInModule": { "filename": "src/processing/lib/spark-job/spark-job-props.ts", "line": 24 }, "name": "schedule", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_events.Schedule" } } ], "symbolId": "src/processing/lib/spark-job/spark-job-props:SparkJobProps" }, "aws-dsf.storage.AccessLogsBucket": { "assembly": "aws-dsf", "base": "aws-cdk-lib.aws_s3.Bucket", "docs": { "example": "const bucket = new dsf.storage.AccessLogsBucket(this, 'AccessLogsBucket')", "remarks": "Default bucket name is `accesslogs---`", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Storage/access-logs-bucket", "stability": "stable", "summary": "Amazon S3 Bucket configured with best-practices and smart defaults for storing S3 access logs." }, "fqn": "aws-dsf.storage.AccessLogsBucket", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/storage/lib/access-logs-bucket.ts", "line": 20 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.BucketProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/storage/lib/access-logs-bucket.ts", "line": 19 }, "name": "AccessLogsBucket", "namespace": "storage", "symbolId": "src/storage/lib/access-logs-bucket:AccessLogsBucket" }, "aws-dsf.storage.AnalyticsBucket": { "assembly": "aws-dsf", "base": "aws-cdk-lib.aws_s3.Bucket", "docs": { "example": "import { Key } from 'aws-cdk-lib/aws-kms';\n\n// Set context value for global data removal policy (or set in cdk.json).\nthis.node.setContext('@data-solutions-framework-on-aws/removeDataOnDestroy', true);\n\nconst encryptionKey = new Key(this, 'DataKey', {\n removalPolicy: cdk.RemovalPolicy.DESTROY,\n enableKeyRotation: true,\n});\n\nnew dsf.storage.AnalyticsBucket(this, 'MyAnalyticsBucket', {\n encryptionKey,\n removalPolicy: cdk.RemovalPolicy.DESTROY,\n});", "remarks": "The default bucket name is `analytics---`", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Storage/analytics-bucket", "stability": "stable", "summary": "Amazon S3 Bucket configured with best-practices and defaults for analytics." }, "fqn": "aws-dsf.storage.AnalyticsBucket", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/storage/lib/analytics-bucket.ts", "line": 37 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "fqn": "aws-dsf.storage.AnalyticsBucketProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/storage/lib/analytics-bucket.ts", "line": 33 }, "name": "AnalyticsBucket", "namespace": "storage", "symbolId": "src/storage/lib/analytics-bucket:AnalyticsBucket" }, "aws-dsf.storage.AnalyticsBucketProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for the `AnalyticsBucket` construct." }, "fqn": "aws-dsf.storage.AnalyticsBucketProps", "kind": "interface", "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 9 }, "name": "AnalyticsBucketProps", "namespace": "storage", "properties": [ { "abstract": true, "docs": { "default": "- If `encryption` is set to `KMS` and this property is undefined,\na new KMS key will be created and associated with this bucket.", "remarks": "The `encryption` property must be either not specified or set to `KMS` or `DSSE`.\nAn error will be emitted if `encryption` is set to `UNENCRYPTED` or `S3_MANAGED`.", "stability": "stable", "summary": "External KMS Key to use for the S3 Bucket encryption." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 121 }, "name": "encryptionKey", "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "default": "- BucketAccessControl.PRIVATE", "stability": "stable", "summary": "Specifies a canned ACL that grants predefined permissions to the bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 166 }, "name": "accessControl", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.BucketAccessControl" } }, { "abstract": true, "docs": { "default": "- False", "remarks": "Requires the `removalPolicy` to be set to `RemovalPolicy.DESTROY`.", "stability": "stable", "summary": "Whether all objects should be automatically deleted when the S3 Bucket is removed from the stack or when the stack is deleted." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 160 }, "name": "autoDeleteObjects", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- CloudFormation defaults will apply. New buckets and objects don't allow public access,\nbut users can modify bucket policies or object permissions to allow public access", "stability": "stable", "summary": "The block public access configuration of this bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 153 }, "name": "blockPublicAccess", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.BlockPublicAccess" } }, { "abstract": true, "docs": { "default": "- False", "remarks": "- If not enabled, every object GET and PUT will cause an API call to KMS (with the\n attendant cost implications of that).\n- If enabled, S3 will use its own time-limited key instead.\n\nOnly relevant, when Encryption is set to `BucketEncryption.KMS` or `BucketEncryption.KMS_MANAGED`.", "stability": "stable", "summary": "Whether Amazon S3 should use its own intermediary key to generate data keys. Only relevant when using KMS for encryption." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 146 }, "name": "bucketKeyEnabled", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- `analytics---`", "stability": "stable", "summary": "The physical name of this S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 133 }, "name": "bucketName", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- No CORS configuration.", "stability": "stable", "summary": "The CORS configuration of this bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 127 }, "name": "cors", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_s3.CorsRule" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- False", "remarks": "S3.5 of the AWS Foundational Security Best Practices Regarding S3.", "stability": "stable", "summary": "Enforces SSL for requests." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 112 }, "name": "enforceSSL", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- False", "stability": "stable", "summary": "Whether this S3 Bucket should send notifications to Amazon EventBridge or not." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 105 }, "name": "eventBridgeEnabled", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- No Intelligent Tiiering Configurations.", "stability": "stable", "summary": "Intelligent Tiering Configurations." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 99 }, "name": "intelligentTieringConfigurations", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_s3.IntelligentTieringConfiguration" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- No inventory configuration", "stability": "stable", "summary": "The inventory configuration of the S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 93 }, "name": "inventories", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_s3.Inventory" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- No lifecycle rules.", "stability": "stable", "summary": "Rules that define how Amazon S3 manages objects during their lifetime." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 87 }, "name": "lifecycleRules", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_s3.LifecycleRule" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- No metrics configuration.", "stability": "stable", "summary": "The metrics configuration of this bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 81 }, "name": "metrics", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_s3.BucketMetrics" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- A new IAM Role will be created.", "stability": "stable", "summary": "The IAM Role to be used by the notifications handler." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 75 }, "name": "notificationsHandlerRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "default": "- No default retention period", "remarks": "Default retention can be configured after a bucket is created if the bucket already\nhas object lock enabled. Enabling object lock for existing buckets is not supported.", "stability": "stable", "summary": "The default retention mode and rules for S3 Object Lock." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 69 }, "name": "objectLockDefaultRetention", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.ObjectLockRetention" } }, { "abstract": true, "docs": { "default": "- False, unless objectLockDefaultRetention is set (then, true)", "remarks": "Enabling object lock for existing buckets is not supported. Object lock must be enabled when the bucket is created.", "stability": "stable", "summary": "Enable object lock on the S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 61 }, "name": "objectLockEnabled", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- No ObjectOwnership configuration, uploading account will own the object.", "stability": "stable", "summary": "The objectOwnership of the S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 54 }, "name": "objectOwnership", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.ObjectOwnership" } }, { "abstract": true, "docs": { "default": "- False", "remarks": "Similar to calling `bucket.grantPublicAccess()`", "stability": "stable", "summary": "Grants public read access to all objects in the S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 48 }, "name": "publicReadAccess", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 41 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- If \"serverAccessLogsPrefix\" undefined - access logs disabled, otherwise - log to current bucket.", "stability": "stable", "summary": "S3 Bucket destination for the server access logs." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 33 }, "name": "serverAccessLogsBucket", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "abstract": true, "docs": { "default": "- No log file prefix", "remarks": "If defined without \"serverAccessLogsBucket\", enables access logs to current S3 Bucket with this prefix.", "stability": "stable", "summary": "Optional log file prefix to use for the S3 Bucket's access logs." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 27 }, "name": "serverAccessLogsPrefix", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- False", "stability": "stable", "summary": "Whether this S3 Bucket should have transfer acceleration turned on or not." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 20 }, "name": "transferAcceleration", "optional": true, "type": { "primitive": "boolean" } }, { "abstract": true, "docs": { "default": "- False (unless object lock is enabled, then true)", "stability": "stable", "summary": "Whether this S3 Bucket should have versioning turned on or not." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/analytics-bucket-props.ts", "line": 14 }, "name": "versioned", "optional": true, "type": { "primitive": "boolean" } } ], "symbolId": "src/storage/lib/analytics-bucket-props:AnalyticsBucketProps" }, "aws-dsf.storage.DataLakeStorage": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "// Set the context value for global data removal policy\nthis.node.setContext('@data-solutions-framework-on-aws/removeDataOnDestroy', true);\n\nnew dsf.storage.DataLakeStorage(this, 'MyDataLakeStorage', {\n bronzeBucketName: 'my-bronze',\n bronzeBucketInfrequentAccessDelay: 90,\n bronzeBucketArchiveDelay: 180,\n silverBucketName: 'my-silver',\n silverBucketInfrequentAccessDelay: 180,\n silverBucketArchiveDelay: 360,\n goldBucketName: 'my-gold',\n goldBucketInfrequentAccessDelay: 180,\n goldBucketArchiveDelay: 360,\n removalPolicy: cdk.RemovalPolicy.DESTROY,\n});", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Storage/data-lake-storage", "stability": "stable", "summary": "Creates the storage layer for a data lake, composed of 3 `AnalyticsBucket` for Bronze, Silver, and Gold data." }, "fqn": "aws-dsf.storage.DataLakeStorage", "initializer": { "docs": { "stability": "stable", "summary": "Construct a new instance of DataLakeStorage." }, "locationInModule": { "filename": "src/storage/lib/data-lake-storage.ts", "line": 66 }, "parameters": [ { "docs": { "summary": "the Scope of the CDK Construct." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the ID of the CDK Construct." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "the DataLakeStorageProps properties." }, "name": "props", "optional": true, "type": { "fqn": "aws-dsf.storage.DataLakeStorageProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/storage/lib/data-lake-storage.ts", "line": 37 }, "methods": [ { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "DataLakeStorage", "namespace": "storage", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The S3 Bucket for access logs." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage.ts", "line": 58 }, "name": "accessLogsBucket", "type": { "fqn": "aws-dsf.storage.AccessLogsBucket" } }, { "docs": { "stability": "stable", "summary": "The S3 Bucket for Bronze layer." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage.ts", "line": 42 }, "name": "bronzeBucket", "type": { "fqn": "aws-dsf.storage.AnalyticsBucket" } }, { "docs": { "stability": "stable", "summary": "The KMS Key used to encrypt all DataLakeStorage S3 buckets." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage.ts", "line": 54 }, "name": "dataLakeKey", "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "docs": { "stability": "stable", "summary": "The S3 Bucket for Gold layer." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage.ts", "line": 50 }, "name": "goldBucket", "type": { "fqn": "aws-dsf.storage.AnalyticsBucket" } }, { "docs": { "stability": "stable", "summary": "The S3 Bucket for Silver layer." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage.ts", "line": 46 }, "name": "silverBucket", "type": { "fqn": "aws-dsf.storage.AnalyticsBucket" } } ], "symbolId": "src/storage/lib/data-lake-storage:DataLakeStorage" }, "aws-dsf.storage.DataLakeStorageProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for the DataLakeStorage Construct." }, "fqn": "aws-dsf.storage.DataLakeStorageProps", "kind": "interface", "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 10 }, "name": "DataLakeStorageProps", "namespace": "storage", "properties": [ { "abstract": true, "docs": { "default": "- Move objects to Glacier after 90 days.", "stability": "stable", "summary": "Delay (in days) before archiving BRONZE data to frozen storage (Glacier storage class)." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 45 }, "name": "bronzeBucketArchiveDelay", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- Move objects to Infrequent Access after 30 days.", "stability": "stable", "summary": "Delay (in days) before moving BRONZE data to cold storage (Infrequent Access storage class)." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 39 }, "name": "bronzeBucketInfrequentAccessDelay", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- `bronze---` will be used.", "remarks": "Use `BucketUtils.generateUniqueBucketName()` to generate a unique name (recommended).", "stability": "stable", "summary": "Name of the Bronze bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 21 }, "name": "bronzeBucketName", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- A single KMS customer key is created.", "stability": "stable", "summary": "The KMS Key used to encrypt all DataLakeStorage S3 buckets." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 15 }, "name": "dataLakeKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "default": "- Objects are not archived to Glacier.", "stability": "stable", "summary": "Delay (in days) before archiving GOLD data to frozen storage (Glacier storage class)." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 70 }, "name": "goldBucketArchiveDelay", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- Move objects to Infrequent Access after 90 days.", "stability": "stable", "summary": "Delay (in days) before moving GOLD data to cold storage (Infrequent Access storage class)." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 64 }, "name": "goldBucketInfrequentAccessDelay", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- `gold---` will be used.", "remarks": "Use `BucketUtils.generateUniqueBucketName()` to generate a unique name (recommended).", "stability": "stable", "summary": "Name of the Gold bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 33 }, "name": "goldBucketName", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise, the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 78 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- Objects are not archived to Glacier.", "stability": "stable", "summary": "Delay (in days) before archiving SILVER data to frozen storage (Glacier storage class)." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 58 }, "name": "silverBucketArchiveDelay", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- Move objects to Infrequent Access after 90 days.", "stability": "stable", "summary": "Delay (in days) before moving SILVER data to cold storage (Infrequent Access storage class)." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 51 }, "name": "silverBucketInfrequentAccessDelay", "optional": true, "type": { "primitive": "number" } }, { "abstract": true, "docs": { "default": "- `silver---` will be used.", "remarks": "Use `BucketUtils.generateUniqueBucketName()` to generate a unique name (recommended).", "stability": "stable", "summary": "Name of the Silver bucket." }, "immutable": true, "locationInModule": { "filename": "src/storage/lib/data-lake-storage-props.ts", "line": 27 }, "name": "silverBucketName", "optional": true, "type": { "primitive": "string" } } ], "symbolId": "src/storage/lib/data-lake-storage-props:DataLakeStorageProps" }, "aws-dsf.utils.ApplicationStackFactory": { "abstract": true, "assembly": "aws-dsf", "docs": { "custom": { "exampleMetadata": "fixture=imports-only" }, "example": "interface MyApplicationStackProps extends cdk.StackProps {\n readonly stage: dsf.utils.CICDStage;\n}\n\nclass MyApplicationStack extends cdk.Stack {\n constructor(scope: Construct, id: string, props?: MyApplicationStackProps) {\n super(scope, id, props);\n // stack logic goes here... and can be customized using props.stage\n }\n}\n\nclass MyApplicationStackFactory extends dsf.utils.ApplicationStackFactory {\n createStack(scope: Construct, stage: dsf.utils.CICDStage): cdk.Stack {\n return new MyApplicationStack(scope, 'MyApplication', {\n stage: stage\n } as MyApplicationStackProps);\n }\n}", "stability": "stable", "summary": "Abstract class that needs to be implemented to pass the application Stack to the CICD pipeline." }, "fqn": "aws-dsf.utils.ApplicationStackFactory", "initializer": { "docs": { "stability": "stable" } }, "kind": "class", "locationInModule": { "filename": "src/utils/lib/application-stack-factory.ts", "line": 32 }, "methods": [ { "abstract": true, "docs": { "stability": "stable", "summary": "Abstract method that needs to be implemented to return the application Stack." }, "locationInModule": { "filename": "src/utils/lib/application-stack-factory.ts", "line": 39 }, "name": "createStack", "parameters": [ { "docs": { "summary": "The scope to create the stack in." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "The stage of the pipeline." }, "name": "stage", "type": { "fqn": "aws-dsf.utils.CICDStage" } } ], "returns": { "type": { "fqn": "aws-cdk-lib.Stack" } } } ], "name": "ApplicationStackFactory", "namespace": "utils", "symbolId": "src/utils/lib/application-stack-factory:ApplicationStackFactory" }, "aws-dsf.utils.ApplicationStage": { "assembly": "aws-dsf", "base": "aws-cdk-lib.Stage", "docs": { "stability": "stable", "summary": "ApplicationStage class that creates a CDK Pipelines Stage from an ApplicationStackFactory." }, "fqn": "aws-dsf.utils.ApplicationStage", "initializer": { "docs": { "stability": "stable", "summary": "Construct a new instance of the SparkCICDStage class." }, "locationInModule": { "filename": "src/utils/lib/application-stage.ts", "line": 57 }, "parameters": [ { "docs": { "summary": "the Scope of the CDK Construct." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the ID of the CDK Construct." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "the ApplicationStage properties." }, "name": "props", "type": { "fqn": "aws-dsf.utils.ApplicationStageProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/utils/lib/application-stage.ts", "line": 44 }, "name": "ApplicationStage", "namespace": "utils", "properties": [ { "docs": { "stability": "stable", "summary": "The list of CfnOutputs created by the CDK Stack." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/application-stage.ts", "line": 49 }, "name": "stackOutputsEnv", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.CfnOutput" }, "kind": "map" } } } ], "symbolId": "src/utils/lib/application-stage:ApplicationStage" }, "aws-dsf.utils.ApplicationStageProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for the `ApplicationStage` class." }, "fqn": "aws-dsf.utils.ApplicationStageProps", "interfaces": [ "aws-cdk-lib.StageProps" ], "kind": "interface", "locationInModule": { "filename": "src/utils/lib/application-stage.ts", "line": 21 }, "name": "ApplicationStageProps", "namespace": "utils", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The application CDK Stack Factory used to create application Stacks." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/application-stage.ts", "line": 25 }, "name": "applicationStackFactory", "type": { "fqn": "aws-dsf.utils.ApplicationStackFactory" } }, { "abstract": true, "docs": { "default": "- No stage is passed to the application stack", "stability": "stable", "summary": "The Stage to deploy the application CDK Stack in." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/application-stage.ts", "line": 37 }, "name": "stage", "type": { "fqn": "aws-dsf.utils.CICDStage" } }, { "abstract": true, "docs": { "default": "- No CfnOutputs are created", "stability": "stable", "summary": "The list of values to create CfnOutputs." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/application-stage.ts", "line": 31 }, "name": "outputsEnv", "optional": true, "type": { "collection": { "elementtype": { "primitive": "string" }, "kind": "map" } } } ], "symbolId": "src/utils/lib/application-stage:ApplicationStageProps" }, "aws-dsf.utils.Architecture": { "assembly": "aws-dsf", "docs": { "stability": "stable", "summary": "List of supported CPU architecture, either X86_64 or ARM64." }, "fqn": "aws-dsf.utils.Architecture", "kind": "enum", "locationInModule": { "filename": "src/utils/lib/architecture.ts", "line": 7 }, "members": [ { "docs": { "stability": "stable" }, "name": "X86_64" }, { "docs": { "stability": "stable" }, "name": "ARM64" } ], "name": "Architecture", "namespace": "utils", "symbolId": "src/utils/lib/architecture:Architecture" }, "aws-dsf.utils.BucketUtils": { "assembly": "aws-dsf", "docs": { "stability": "stable", "summary": "Utils for working with Amazon S3 Buckets." }, "fqn": "aws-dsf.utils.BucketUtils", "initializer": { "docs": { "stability": "stable" } }, "kind": "class", "locationInModule": { "filename": "src/utils/lib/bucket-utils.ts", "line": 11 }, "methods": [ { "docs": { "remarks": "The bucket name is suffixed the AWS account ID, the AWS region and a unique 8 characters hash.\nThe maximum length for name is 26 characters.", "returns": "the unique Name for the bucket", "stability": "stable", "summary": "Generate a unique Amazon S3 bucket name based on the provided name, CDK construct ID and CDK construct scope." }, "locationInModule": { "filename": "src/utils/lib/bucket-utils.ts", "line": 22 }, "name": "generateUniqueBucketName", "parameters": [ { "docs": { "summary": "the current scope where the construct is created (generally `this`)." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the CDK ID of the construct." }, "name": "id", "type": { "primitive": "string" } }, { "docs": { "summary": "the name of the bucket." }, "name": "name", "type": { "primitive": "string" } } ], "returns": { "type": { "primitive": "string" } }, "static": true } ], "name": "BucketUtils", "namespace": "utils", "symbolId": "src/utils/lib/bucket-utils:BucketUtils" }, "aws-dsf.utils.CICDStage": { "assembly": "aws-dsf", "docs": { "stability": "stable", "summary": "The list of CICD Stages used in CICD Pipelines." }, "fqn": "aws-dsf.utils.CICDStage", "kind": "enum", "locationInModule": { "filename": "src/utils/lib/application-stage.ts", "line": 13 }, "members": [ { "docs": { "stability": "stable" }, "name": "STAGING" }, { "docs": { "stability": "stable" }, "name": "PROD" } ], "name": "CICDStage", "namespace": "utils", "symbolId": "src/utils/lib/application-stage:CICDStage" }, "aws-dsf.utils.DataVpc": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "const vpc = new dsf.utils.DataVpc(this, 'DataVpc', {\n vpcCidr: '10.0.0.0/16',\n});\n\nvpc.tagVpc('Name', 'My VPC');", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Utils/data-vpc", "stability": "stable", "summary": "Creates a VPC with best practices for securely deploying data solutions." }, "fqn": "aws-dsf.utils.DataVpc", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/utils/lib/data-vpc.ts", "line": 48 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "fqn": "aws-dsf.utils.DataVpcProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/utils/lib/data-vpc.ts", "line": 25 }, "methods": [ { "docs": { "stability": "stable", "summary": "Tag the VPC and the subnets." }, "locationInModule": { "filename": "src/utils/lib/data-vpc.ts", "line": 137 }, "name": "tagVpc", "parameters": [ { "docs": { "summary": "the tag key." }, "name": "key", "type": { "primitive": "string" } }, { "docs": { "summary": "the tag value." }, "name": "value", "type": { "primitive": "string" } } ] } ], "name": "DataVpc", "namespace": "utils", "properties": [ { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group created for the VPC flow logs." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc.ts", "line": 42 }, "name": "flowLogGroup", "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The KMS Key used to encrypt VPC flow logs." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc.ts", "line": 34 }, "name": "flowLogKey", "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "docs": { "stability": "stable", "summary": "The IAM role used to publish VPC Flow Logs." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc.ts", "line": 38 }, "name": "flowLogRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The S3 VPC endpoint gateway." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc.ts", "line": 46 }, "name": "s3VpcEndpoint", "type": { "fqn": "aws-cdk-lib.aws_ec2.IGatewayVpcEndpoint" } }, { "docs": { "stability": "stable", "summary": "The amazon VPC created." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc.ts", "line": 30 }, "name": "vpc", "type": { "fqn": "aws-cdk-lib.aws_ec2.IVpc" } } ], "symbolId": "src/utils/lib/data-vpc:DataVpc" }, "aws-dsf.utils.DataVpcProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "The properties for the `DataVpc` construct." }, "fqn": "aws-dsf.utils.DataVpcProps", "kind": "interface", "locationInModule": { "filename": "src/utils/lib/data-vpc-props.ts", "line": 13 }, "name": "DataVpcProps", "namespace": "utils", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The CIDR to use to create the subnets in the VPC." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc-props.ts", "line": 17 }, "name": "vpcCidr", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- A new KMS key is created", "remarks": "The resource policy of the key must be configured according to the AWS documentation.", "see": "https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html", "stability": "stable", "summary": "The KMS key used to encrypt the VPC Flow Logs in the CloudWatch Log Group." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc-props.ts", "line": 24 }, "name": "flowLogKey", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_kms.IKey" } }, { "abstract": true, "docs": { "default": "- One week retention", "stability": "stable", "summary": "The retention period to apply to VPC Flow Logs." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc-props.ts", "line": 36 }, "name": "flowLogRetention", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_logs.RetentionDays" } }, { "abstract": true, "docs": { "default": "- A new IAM role is created", "remarks": "The role must be configured as described in the AWS VPC Flow Log documentation.", "see": "https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-cwl.html#flow-logs-iam-role", "stability": "stable", "summary": "The IAM Role used to send the VPC Flow Logs in CloudWatch." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc-props.ts", "line": 31 }, "name": "flowLogRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise, the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/data-vpc-props.ts", "line": 43 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } } ], "symbolId": "src/utils/lib/data-vpc-props:DataVpcProps" }, "aws-dsf.utils.S3DataCopy": { "assembly": "aws-dsf", "base": "constructs.Construct", "docs": { "example": "import { Bucket } from 'aws-cdk-lib/aws-s3';\n\nconst sourceBucket = Bucket.fromBucketName(this, 'SourceBucket', 'nyc-tlc');\nconst bucketName = `test-${this.region}-${this.account}-${dsf.utils.Utils.generateUniqueHash(this, 'TargetBucket')}`;\n\nconst targetBucket = new Bucket(this, 'TargetBucket');\n\nnew dsf.utils.S3DataCopy(this, 'S3DataCopy', {\n sourceBucket,\n sourceBucketPrefix: 'trip data/',\n sourceBucketRegion: 'us-east-1',\n targetBucket,\n});", "see": "https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Utils/s3-data-copy", "stability": "stable", "summary": "Copy data from one S3 bucket to another." }, "fqn": "aws-dsf.utils.S3DataCopy", "initializer": { "docs": { "stability": "stable" }, "locationInModule": { "filename": "src/utils/lib/s3-data-copy.ts", "line": 68 }, "parameters": [ { "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "name": "id", "type": { "primitive": "string" } }, { "name": "props", "type": { "fqn": "aws-dsf.utils.S3DataCopyProps" } } ] }, "kind": "class", "locationInModule": { "filename": "src/utils/lib/s3-data-copy.ts", "line": 37 }, "methods": [ { "docs": { "stability": "stable", "summary": "Retrieve DSF package.json version." }, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 80 }, "name": "retrieveVersion", "returns": { "type": { "primitive": "any" } } } ], "name": "S3DataCopy", "namespace": "utils", "properties": [ { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 20 }, "name": "DSF_OWNED_TAG", "static": true, "type": { "primitive": "string" } }, { "const": true, "docs": { "stability": "stable" }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/tracked-construct.ts", "line": 18 }, "name": "DSF_TRACKING_CODE", "static": true, "type": { "primitive": "string" } }, { "docs": { "stability": "stable", "summary": "The Lambda Function for the copy." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy.ts", "line": 50 }, "name": "copyFunction", "type": { "fqn": "aws-cdk-lib.aws_lambda.IFunction" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group for the S3 data copy." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy.ts", "line": 42 }, "name": "copyLogGroup", "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The IAM Role for the copy Lambba Function." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy.ts", "line": 46 }, "name": "copyRole", "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The Lambda function for the S3 data copy cleaning up lambda." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy.ts", "line": 62 }, "name": "cleanUpFunction", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_lambda.IFunction" } }, { "docs": { "stability": "stable", "summary": "The CloudWatch Log Group for the S3 data copy cleaning up lambda." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy.ts", "line": 58 }, "name": "cleanUpLogGroup", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_logs.ILogGroup" } }, { "docs": { "stability": "stable", "summary": "The IAM Role for the the S3 data copy cleaning up lambda." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy.ts", "line": 66 }, "name": "cleanUpRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "docs": { "stability": "stable", "summary": "The list of EC2 Security Groups used by the Lambda Functions." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy.ts", "line": 54 }, "name": "securityGroups", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_ec2.ISecurityGroup" }, "kind": "array" } } } ], "symbolId": "src/utils/lib/s3-data-copy:S3DataCopy" }, "aws-dsf.utils.S3DataCopyProps": { "assembly": "aws-dsf", "datatype": true, "docs": { "stability": "stable", "summary": "Properties for S3DataCopy construct." }, "fqn": "aws-dsf.utils.S3DataCopyProps", "kind": "interface", "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 12 }, "name": "S3DataCopyProps", "namespace": "utils", "properties": [ { "abstract": true, "docs": { "stability": "stable", "summary": "The source S3 Bucket containing the data to copy." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 17 }, "name": "sourceBucket", "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The source S3 Bucket region." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 26 }, "name": "sourceBucketRegion", "type": { "primitive": "string" } }, { "abstract": true, "docs": { "stability": "stable", "summary": "The target S3 Bucket." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 30 }, "name": "targetBucket", "type": { "fqn": "aws-cdk-lib.aws_s3.IBucket" } }, { "abstract": true, "docs": { "default": "- A new role is created", "stability": "stable", "summary": "The IAM Role to use in the custom resource for copying data." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 40 }, "name": "executionRole", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_iam.IRole" } }, { "abstract": true, "docs": { "default": "- The resources are not deleted (`RemovalPolicy.RETAIN`).", "remarks": "If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true.\nOtherwise, the removalPolicy is reverted to RETAIN.", "stability": "stable", "summary": "The removal policy when deleting the CDK resource." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 63 }, "name": "removalPolicy", "optional": true, "type": { "fqn": "aws-cdk-lib.RemovalPolicy" } }, { "abstract": true, "docs": { "default": "- If `vpc` is not supplied, no security groups are attached. Otherwise, a dedicated security\ngroup is created for each function.", "stability": "stable", "summary": "The list of security groups to attach to the custom resource." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 56 }, "name": "securityGroups", "optional": true, "type": { "collection": { "elementtype": { "fqn": "aws-cdk-lib.aws_ec2.ISecurityGroup" }, "kind": "array" } } }, { "abstract": true, "docs": { "default": "- No prefix is used", "stability": "stable", "summary": "The source bucket prefix with a slash at the end." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 22 }, "name": "sourceBucketPrefix", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- The Custom Resource is executed in VPCs owned by AWS Lambda service.", "stability": "stable", "summary": "The subnets to deploy the custom resource in." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 50 }, "name": "subnets", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.SubnetSelection" } }, { "abstract": true, "docs": { "default": "- No prefix is used", "stability": "stable", "summary": "The target S3 Bucket prefix with a slash at the end." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 35 }, "name": "targetBucketPrefix", "optional": true, "type": { "primitive": "string" } }, { "abstract": true, "docs": { "default": "- The Custom Resource is executed in VPCs owned by AWS Lambda service.", "stability": "stable", "summary": "The VPC to deploy the custom resource in." }, "immutable": true, "locationInModule": { "filename": "src/utils/lib/s3-data-copy-props.ts", "line": 45 }, "name": "vpc", "optional": true, "type": { "fqn": "aws-cdk-lib.aws_ec2.IVpc" } } ], "symbolId": "src/utils/lib/s3-data-copy-props:S3DataCopyProps" }, "aws-dsf.utils.StepFunctionUtils": { "assembly": "aws-dsf", "docs": { "stability": "stable", "summary": "Utils for working with AWS Step Functions." }, "fqn": "aws-dsf.utils.StepFunctionUtils", "initializer": { "docs": { "stability": "stable" } }, "kind": "class", "locationInModule": { "filename": "src/utils/lib/step-function-utils.ts", "line": 7 }, "methods": [ { "docs": { "returns": "config converted to pascal case.", "stability": "stable", "summary": "Convert camel case properties to pascal case as required by AWS Step Functions API." }, "locationInModule": { "filename": "src/utils/lib/step-function-utils.ts", "line": 14 }, "name": "camelToPascal", "parameters": [ { "name": "config", "type": { "collection": { "elementtype": { "primitive": "any" }, "kind": "map" } } } ], "returns": { "type": { "collection": { "elementtype": { "primitive": "any" }, "kind": "map" } } }, "static": true } ], "name": "StepFunctionUtils", "namespace": "utils", "symbolId": "src/utils/lib/step-function-utils:StepFunctionUtils" }, "aws-dsf.utils.Utils": { "assembly": "aws-dsf", "docs": { "stability": "stable", "summary": "Utilities class used across the different resources." }, "fqn": "aws-dsf.utils.Utils", "initializer": { "docs": { "stability": "stable" } }, "kind": "class", "locationInModule": { "filename": "src/utils/lib/utils.ts", "line": 14 }, "methods": [ { "docs": { "returns": "the hash", "stability": "stable", "summary": "Generate an 8 character hash from a string based on HMAC algorithm." }, "locationInModule": { "filename": "src/utils/lib/utils.ts", "line": 82 }, "name": "generateHash", "parameters": [ { "docs": { "summary": "the text to hash." }, "name": "text", "type": { "primitive": "string" } } ], "returns": { "type": { "primitive": "string" } }, "static": true }, { "docs": { "returns": "the hash", "stability": "stable", "summary": "Generate a unique hash of 8 characters from the CDK scope using its path and the stack name." }, "locationInModule": { "filename": "src/utils/lib/utils.ts", "line": 95 }, "name": "generateUniqueHash", "parameters": [ { "docs": { "summary": "the CDK construct scope." }, "name": "scope", "type": { "fqn": "constructs.Construct" } }, { "docs": { "summary": "the CDK ID of the construct." }, "name": "id", "optional": true, "type": { "primitive": "string" } } ], "returns": { "type": { "primitive": "string" } }, "static": true }, { "docs": { "stability": "stable", "summary": "Take a document stored as string and load it as YAML." }, "locationInModule": { "filename": "src/utils/lib/utils.ts", "line": 50 }, "name": "loadYaml", "parameters": [ { "docs": { "summary": "the document stored as string." }, "name": "document", "type": { "primitive": "string" } } ], "returns": { "type": { "primitive": "any" } }, "static": true }, { "docs": { "stability": "stable", "summary": "Create a random string to be used as a seed for IAM User password." }, "locationInModule": { "filename": "src/utils/lib/utils.ts", "line": 28 }, "name": "randomize", "parameters": [ { "docs": { "summary": "the string to which to append a random string." }, "name": "name", "type": { "primitive": "string" } } ], "returns": { "type": { "primitive": "string" } }, "static": true }, { "docs": { "stability": "stable", "summary": "Read a YAML file from the path provided and return it." }, "locationInModule": { "filename": "src/utils/lib/utils.ts", "line": 36 }, "name": "readYamlDocument", "parameters": [ { "docs": { "summary": "the path to the file." }, "name": "path", "type": { "primitive": "string" } } ], "returns": { "type": { "primitive": "string" } }, "static": true }, { "docs": { "stability": "stable", "summary": "Sanitize a string by removing upper case and replacing special characters except underscore." }, "locationInModule": { "filename": "src/utils/lib/utils.ts", "line": 20 }, "name": "stringSanitizer", "parameters": [ { "docs": { "summary": "the string to sanitize." }, "name": "toSanitize", "type": { "primitive": "string" } } ], "returns": { "type": { "primitive": "string" } }, "static": true }, { "docs": { "returns": "the string in Pascal case", "stability": "stable", "summary": "Convert a string to PascalCase." }, "locationInModule": { "filename": "src/utils/lib/utils.ts", "line": 59 }, "name": "toPascalCase", "parameters": [ { "docs": { "summary": "the string to convert to PascalCase." }, "name": "text", "type": { "primitive": "string" } } ], "returns": { "type": { "primitive": "string" } }, "static": true } ], "name": "Utils", "namespace": "utils", "symbolId": "src/utils/lib/utils:Utils" } }, "version": "1.0.0-rc.9", "fingerprint": "OZQaXuyG+by08pQ9rRw4PmSkrlVtgl+Da+bbEOfGngE=" }