import { given } from "@nivinjoseph/n-defensive"; import { ArgumentException } from "@nivinjoseph/n-exception"; import * as Pulumi from "@pulumi/pulumi"; import type { VpcDetails } from "../vpc/vpc-details.js"; import type { AppClusterConfig, AppConfig, AppConfigShape } from "./app-config.js"; // import { ManagedPolicy, Policy, Role } from "@pulumi/aws/iam"; import * as aws from "@pulumi/aws"; // import * as awsx from "@pulumi/awsx"; import type { EnvVar } from "../common/env-var.js"; import { NfraConfig } from "../common/nfra-config.js"; import type { Secret } from "../secret/secret.js"; import type { PolicyDocument } from "../security/policy/policy-document.js"; import type { AppClusterDetails } from "./app-cluster-details.js"; import type { AppDetails } from "./app-details.js"; // import { LogConfiguration } from "@pulumi/aws/ecs"; // import { VirtualNode } from "@pulumi/aws/appmesh"; import { AppComputeProfile } from "./app-compute-profile.js"; import { DescribeImagesCommand, ECRClient } from "@aws-sdk/client-ecr"; import { Logger } from "../common/logger.js"; import * as crypto from "node:crypto"; import { enumValueList, enumValues } from "../common/validation-helper.js"; export abstract class AppProvisioner { /** * @description Liveness-grade ECS health check timing for app containers. A task is only marked UNHEALTHY after * `retries` consecutive failures, so a busy-but-responding task is never killed while a wedged one is still * replaced (worst case ~ retries x timeout + (retries - 1) x interval ~ 8.7 min). * ECS bounds: interval 5-300, timeout 2-60, retries 1-10, startPeriod 0-300. */ private static readonly _healthCheckIntervalSeconds = 30; private static readonly _healthCheckTimeoutSeconds = 30; private static readonly _healthCheckRetries = 10; private static readonly _healthCheckStartPeriodSeconds = 60; // target tracking owns steady-state scaling and all scale-in private static readonly _cpuTargetPercent = 45; private static readonly _scaleOutCooldownSeconds = 30; private static readonly _scaleInCooldownSeconds = 300; // step scale-out reacts to a single 1-minute datapoint of the hottest task, well above the target tracking target private static readonly _stepScaleOutCpuThresholdPercent = 75; private static readonly _stepScaleOutSecondTierOffsetPercent = 15; // second step from threshold + 15 => 90 % private static readonly _stepScaleOutCooldownSeconds = 60; private static readonly _stepScaleOutAlarmPeriodSeconds = 60; // AWS/ECS CPUUtilization is published per minute // images from these registries are used verbatim and are never looked up in the account's private ECR private static readonly _publicImageRegistries = ["docker.", "public.ecr."]; private readonly _name: string; private readonly _config: T; private readonly _version: string; private readonly _appEnv: string; protected get name(): string { return this._name; } protected get vpcDetails(): VpcDetails { return this._config.vpcDetails; } protected get config(): T { return this._config; } protected get version(): string { return this._version; } protected get hasDatadog(): boolean { return this._config.datadogConfig != null; } protected constructor(name: string, config: T) { given(name, "name").ensureHasValue().ensureIsString(); // this._name = CommonHelper.prefixName(name); this._name = name; // copy so the caller's object is never mutated, then fill defaults (an explicit undefined also gets the default) config = { ...config }; config.computeProfile ??= AppComputeProfile.small; config.minCapacity ??= 1; config.maxCapacity ??= 1; config.isOn ??= true; given(config as AppConfigShape, "config").ensureHasValue().ensureIsObject() .ensureHasStructure({ vpcDetails: "object", subnetNamePrefix: "string", computeProfile: "number", "customCompute?": { cpu: "number", memory: "number" }, image: "string", "version?": "string", "command?": ["string"], "entryPoint?": ["string"], "useDockerfileCommandOrEntryPoint?": "boolean", "envVars?": ["object"], "secrets?": ["object"], "policies?": "array", isOn: "boolean", "datadogConfig?": "object", "enableXray?": "boolean", "clusterConfig?": "object", "cluster?": "object", "minCapacity?": "number", "maxCapacity?": "number", "disableReadonlyRootFilesystem?": "boolean", "cpuArchitecture?": "string", "tags?": "object" }) // .ensure(t => t.image.contains(":v"), "config.image does not have a valid tag") .ensureWhen(config.policies != null && config.policies.isNotEmpty && config.policies.some(t => typeof t === "string"), (t) => t.policies!.where(u => typeof u === "string").every(u => (u).startsWith("arn:aws:iam::aws:policy/")), "policy string values must be aws managed policies") .ensureWhen(config.datadogConfig != null, (t) => !t.enableXray, "only one of datadogConfig or enableXray can be used") .ensure(t => !(t.clusterConfig == null && t.cluster == null), "one of clusterConfig or cluster must be provided") .ensureWhen(!config.useDockerfileCommandOrEntryPoint, t => !((t.command == null || t.command.isEmpty) && (t.entryPoint == null || t.entryPoint.isEmpty)), "one of either command or entryPoint must be provided") .ensureWhen(!config.useDockerfileCommandOrEntryPoint, t => !(t.command != null && t.entryPoint != null), "only one of either command or entryPoint must be provided") .ensure(t => t.minCapacity != null && t.minCapacity >= 0 && t.minCapacity <= 50, "minCapacity must be between 0 and 50") .ensure(t => t.maxCapacity != null && t.maxCapacity >= 0 && t.maxCapacity <= 50, "maxCapacity must be between 0 and 50") .ensure(t => t.minCapacity! <= t.maxCapacity!, "minCapacity must be <= maxCapacity") .ensureWhen(config.cpuArchitecture != null, t => ["X86_64", "ARM64"].contains(t.cpuArchitecture!), "cpuArchitecture must be one of X86_64 or ARM64") .ensure(t => enumValues(AppComputeProfile).contains(t.computeProfile!), `computeProfile must be one of ${enumValueList(AppComputeProfile)}`) .ensureWhen(config.cluster?.usesSpotInstances === true || config.clusterConfig?.useSpotCapacity != null, t => t.isOn === false || t.minCapacity === t.maxCapacity, "cluster uses spot instances, cannot enable autoscaling (minCapacity must equal maxCapacity)"); config.enableXray ??= false; this._config = config; if (this._config.isOn === false) this._config.minCapacity = this._config.maxCapacity = 0; if (this._config.version != null) { this._version = this._config.version; } else { if (this._config.image.contains(":v")) this._version = this._config.image.split(":v").takeLast(); else this._version = "UNKNOWN"; } // defaulting cpu architecture to ARM64 this._config.cpuArchitecture ??= "ARM64"; this._appEnv = NfraConfig.appEnv; } public static provisionAppCluster(name: string, config: AppClusterConfig, vpcDetails: VpcDetails): AppClusterDetails { given(name, "name").ensureHasValue().ensureIsString(); // name = CommonHelper.prefixName(name); given(config, "config").ensureHasValue().ensureIsObject().ensureHasStructure({ "enableContainerInsights?": "boolean", "useSpotCapacity?": { "onlyUseSpotCapacity?": "boolean" } }); given(vpcDetails, "vpcDetails").ensureHasValue().ensureIsObject(); const clusterName = `${name}-cls`; const cluster = new aws.ecs.Cluster(clusterName, { // capacityProviders: ["FARGATE"], // deprecated serviceConnectDefaults: { namespace: vpcDetails.privateDnsNamespace.arn }, settings: [ { name: "containerInsights", value: config.enableContainerInsights ? "enabled" : "disabled" } ], tags: { ...NfraConfig.tags, Name: clusterName } }); let capacityProviders = [CapacityProvider.fargate]; let capacityProviderStrategies : Array = [{ capacityProvider: CapacityProvider.fargate, base: 1, weight: 100 }]; if (config.useSpotCapacity != null) { capacityProviders.push(CapacityProvider.fargateSpot); if (config.useSpotCapacity.onlyUseSpotCapacity) { capacityProviders = capacityProviders .where(t => t !== CapacityProvider.fargate); capacityProviderStrategies = capacityProviderStrategies .where(t => t.capacityProvider !== CapacityProvider.fargate); capacityProviderStrategies.push({ capacityProvider: CapacityProvider.fargateSpot, base: 1, weight: 100 }); } else { capacityProviderStrategies = capacityProviderStrategies .where(t => t.capacityProvider !== CapacityProvider.fargate); capacityProviderStrategies.push({ capacityProvider: CapacityProvider.fargate, base: 1, weight: 1 }, { capacityProvider: CapacityProvider.fargateSpot, weight: 99 }); } } const clusterCapacityProviderName = `${name}-cls-cp`; new aws.ecs.ClusterCapacityProviders(clusterCapacityProviderName, { clusterName: cluster.name, capacityProviders, defaultCapacityProviderStrategies: capacityProviderStrategies }); return { clusterName: cluster.name, clusterArn: cluster.arn, usesSpotInstances: config.useSpotCapacity != null }; } public async provision(): Promise { await this._verifyImageExists(); return this.provisionApp(); } protected abstract provisionApp(): U; protected createAppCluster(): AppClusterDetails { return this._config.cluster ?? AppProvisioner.provisionAppCluster( this._name, this._config.clusterConfig, this.vpcDetails); } protected createExecutionRole(policies?: ReadonlyArray): Pulumi.Output { // given(isEc2, "isEc2").ensureHasValue().ensureIsBoolean(); given(policies, "policies").ensureIsArray().ensureIsNotEmpty(); const taskExecutionPolicies = new Array(); if (policies != null && policies.isNotEmpty) taskExecutionPolicies.push(...policies); const managedPolicies = taskExecutionPolicies .where(t => typeof t === "string") as Array; const policyDocs = taskExecutionPolicies .where(t => typeof t !== "string") as Array; const secrets = new Array(); if (this._config.secrets != null && this._config.secrets.isNotEmpty) secrets.push(...this._config.secrets); if (this._config.datadogConfig != null) secrets.push(this._config.datadogConfig.apiKey); const policyArns = [ ...managedPolicies, "arn:aws:iam::aws:policy/AmazonECS_FullAccess", "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", // aws.iam.ManagedPolicy.CloudWatchFullAccess "arn:aws:iam::aws:policy/CloudWatchFullAccessV2" ]; const resolvedPolicyArns = new Array>(); if (!secrets.isEmpty) { const secretPolicyName = `${this._name}-sct-tp`; const secretPolicy = new aws.iam.Policy(secretPolicyName, { policy: { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": "secretsmanager:GetSecretValue", "Resource": secrets.map(t => t.arn) } ] }, tags: { ...NfraConfig.tags, Name: secretPolicyName } }); resolvedPolicyArns.push(secretPolicy.arn); } const createdPolicies = policyDocs.map((policyDoc, index) => { const policyName = `${this._name}-tep-${index}`; const policy = new aws.iam.Policy(policyName, { policy: policyDoc, tags: { ...NfraConfig.tags, Name: policyName } }); resolvedPolicyArns.push(policy.arn); return policy; }); // const factoryFunc = isEc2 // // eslint-disable-next-line @typescript-eslint/unbound-method // ? awsx.ecs.EC2TaskDefinition.createExecutionRole // // eslint-disable-next-line @typescript-eslint/unbound-method // : awsx.ecs.FargateTaskDefinition.createExecutionRole; if (resolvedPolicyArns.isEmpty) return Pulumi.output( _createRole(`${this._name}-ter`, _defaultRoleAssumeRolePolicy(), policyArns, { dependsOn: createdPolicies }) ); else return Pulumi.all(resolvedPolicyArns).apply(resolvedArns => { return _createRole( `${this._name}-ter`, _defaultRoleAssumeRolePolicy(), [ ...policyArns, ...resolvedArns ], { dependsOn: createdPolicies }); }); } protected createTaskRole(isEc2 = false, policies?: ReadonlyArray): Pulumi.Output { given(isEc2, "isEc2").ensureHasValue().ensureIsBoolean(); given(policies, "policies").ensureIsArray().ensureIsNotEmpty(); const taskPolicies = new Array(); if (policies != null && policies.isNotEmpty) taskPolicies.push(...policies); if (this._config.policies != null && this._config.policies.isNotEmpty) taskPolicies.push(...this._config.policies); const managedPolicies = taskPolicies .where(t => typeof t === "string") as Array; const policyDocs = taskPolicies .where(t => typeof t !== "string") as Array; const policyArns = [ ...managedPolicies, // aws.iam.ManagedPolicy.CloudWatchFullAccess, "arn:aws:iam::aws:policy/CloudWatchFullAccessV2", // "arn:aws:iam::aws:policy/AWSAppMeshEnvoyAccess", aws.iam.ManagedPolicy.AmazonECSFullAccess, ...this._config.enableXray ? [aws.iam.ManagedPolicy.AWSXRayDaemonWriteAccess] : [] ]; const resolvedPolicyArns = new Array>(); const createdPolicies = policyDocs.map((policyDoc, index) => { const policyName = `${this._name}-tp-${index}`; const policy = new aws.iam.Policy(policyName, { policy: policyDoc, tags: { ...NfraConfig.tags, Name: policyName } }); resolvedPolicyArns.push(policy.arn); return policy; }); // const factoryFunc = isEc2 // // eslint-disable-next-line @typescript-eslint/unbound-method // ? awsx.ecs.EC2TaskDefinition.createTaskRole // // eslint-disable-next-line @typescript-eslint/unbound-method // : awsx.ecs.FargateTaskDefinition.createTaskRole; if (resolvedPolicyArns.isEmpty) return Pulumi.output( _createRole(`${this._name}-tr`, _defaultRoleAssumeRolePolicy(), policyArns, { dependsOn: createdPolicies }) ); else return Pulumi.all(resolvedPolicyArns).apply(resolvedArns => { return _createRole( `${this._name}-tr`, _defaultRoleAssumeRolePolicy(), [ ...policyArns, ...resolvedArns ], { dependsOn: createdPolicies }); }); } protected createAppContainer(): aws.ecs.ContainerDefinition { return { name: this._name, image: this._isPublicImage() ? this._config.image : `${NfraConfig.ecrBase}/${this.config.image}`, essential: true, readonlyRootFilesystem: this._config.disableReadonlyRootFilesystem ? false : true, cpu: 0, command: this.config.command ? [...this.config.command] : undefined, entryPoint: this.config.entryPoint ? [...this.config.entryPoint] : undefined, portMappings: [], environment: [ ...this.config.envVars != null ? this.config.envVars : [], ...this._createInstrumentationEnvironmentVariables() ].reverse() .distinct(t => t.name) .orderBy(t => t.name) .map(t => { // if (typeof t.value === "string") // return { name: t.name, value: Pulumi.output(t.value) }; // return t; return { name: t.name, value: this._interpolatify(t.value) }; }), // .map(t => (>t.value) // .apply(u => ({ name: t.name, value: u }))), secrets: this.config.secrets && this.config.secrets.isNotEmpty ? this.config.secrets .orderBy(t => t.name) // .map(t => t.arn.apply(u => ({ name: t.name, valueFrom: u }))) .map(t => ({ name: t.name, valueFrom: t.arn })) : undefined, stopTimeout: 45, logConfiguration: this._createLogConfiguration(), dockerLabels: this.hasDatadog ? { ...this._createDatadogInstrumentationLabels() } : undefined, mountPoints: [], volumesFrom: [], dependsOn: [] }; } protected createContainerDefinitions( // virtualNode: aws.appmesh.VirtualNode, appContainerOverrides?: Partial, isEc2 = false): Pulumi.Output { // given(virtualNode, "virtualNode").ensureHasValue().ensureIsObject(); given(appContainerOverrides, "appContainerOverrides").ensureIsObject(); return this._stringifyContainerDefinitions({ [this._name]: appContainerOverrides != null ? { ...this.createAppContainer(), ...appContainerOverrides } : this.createAppContainer(), ...this._createInstrumentationContainers( // virtualNode, isEc2) }) // FIXME: use only to debug container definitions json // .apply(t => // { // console.log(t); // return t; // }) ; } protected createTaskVolumeConfiguration(isEc2 = false, additionalVolumes?: ReadonlyArray): Array { given(isEc2, "isEc2").ensureHasValue().ensureIsBoolean(); given(additionalVolumes, "additionalVolumes").ensureIsArray().ensureIsNotEmpty(); const taskVolumeConfiguration: Array = additionalVolumes != null ? [...additionalVolumes] : []; if (this.hasDatadog) { taskVolumeConfiguration.push({ "name": "etc-datadog_agent" }); if (isEc2) { taskVolumeConfiguration.push( { name: "docker_sock", hostPath: "/var/run/docker.sock" }, { name: "cgroup", hostPath: "/sys/fs/cgroup/" }, { name: "proc", hostPath: "/proc/" } ); } } return taskVolumeConfiguration; } /** * @description Builds the app container's ECS health check. `probeBudgetSeconds` is the longest the probe command * itself may run before giving up; it must finish before ECS's own timeout so the probe result, not an ECS kill, * decides the outcome. */ protected createAppHealthCheck(shellCommand: string, probeBudgetSeconds: number): aws.ecs.HealthCheck { given(shellCommand, "shellCommand").ensureHasValue().ensureIsString(); given(probeBudgetSeconds, "probeBudgetSeconds").ensureHasValue().ensureIsNumber() .ensure(t => t > 0 && t < AppProvisioner._healthCheckTimeoutSeconds, `probe budget must be less than the ECS health check timeout of ${AppProvisioner._healthCheckTimeoutSeconds}s`); return { command: ["CMD-SHELL", shellCommand], interval: AppProvisioner._healthCheckIntervalSeconds, timeout: AppProvisioner._healthCheckTimeoutSeconds, retries: AppProvisioner._healthCheckRetries, startPeriod: AppProvisioner._healthCheckStartPeriodSeconds }; } protected supportsAutoScaling(): boolean { return this._config.minCapacity! < this._config.maxCapacity!; } protected configureAutoScaling(cluster: AppClusterDetails, service: aws.ecs.Service): void { if (!this.supportsAutoScaling()) return; given(cluster, "cluster").ensureHasValue().ensureIsObject() .ensure(t => !t.usesSpotInstances, "cluster uses spot instances, cannot enable autoscaling"); const asTarget = new aws.appautoscaling.Target(`${this._name}-ast`, { minCapacity: this.config.minCapacity!, maxCapacity: this.config.maxCapacity!, resourceId: Pulumi.interpolate`service/${cluster.clusterName}/${service.name}`, scalableDimension: "ecs:service:DesiredCount", serviceNamespace: "ecs" }); new aws.appautoscaling.Policy(`${this._name}-asp`, { policyType: "TargetTrackingScaling", resourceId: asTarget.resourceId, scalableDimension: asTarget.scalableDimension, serviceNamespace: asTarget.serviceNamespace, targetTrackingScalingPolicyConfiguration: { targetValue: AppProvisioner._cpuTargetPercent, scaleInCooldown: AppProvisioner._scaleInCooldownSeconds, scaleOutCooldown: AppProvisioner._scaleOutCooldownSeconds, predefinedMetricSpecification: { predefinedMetricType: "ECSServiceAverageCPUUtilization" } } }); // Scale-OUT-only step policy. Target tracking needs ~3 consecutive 1-minute datapoints above target, which loses // the race against the container health check when one task saturates; this alarm fires on a single datapoint. // It uses the Maximum statistic: ECS publishes one sample per task into the service metric, so Maximum is the // hottest task and one saturated task is caught at any desiredCount, where the service Average would dilute it. // Scale-in stays with target tracking: AWS resolves concurrent scale-out requests to the largest capacity, and // mixing step and target tracking only conflicts on scale-in. const stepPolicyName = `${this._name}-asp-step`; const stepPolicy = new aws.appautoscaling.Policy(stepPolicyName, { policyType: "StepScaling", resourceId: asTarget.resourceId, scalableDimension: asTarget.scalableDimension, serviceNamespace: asTarget.serviceNamespace, stepScalingPolicyConfiguration: { adjustmentType: "ChangeInCapacity", cooldown: AppProvisioner._stepScaleOutCooldownSeconds, metricAggregationType: "Maximum", // bounds are offsets from the alarm threshold (lower inclusive, upper exclusive above threshold): // (-inf, threshold + 15) => +1 task, [threshold + 15, +inf) => +2 tasks stepAdjustments: [ { metricIntervalUpperBound: `${AppProvisioner._stepScaleOutSecondTierOffsetPercent}`, scalingAdjustment: 1 }, { metricIntervalLowerBound: `${AppProvisioner._stepScaleOutSecondTierOffsetPercent}`, scalingAdjustment: 2 } ] } }); const stepAlarmName = `${stepPolicyName}-alm`; new aws.cloudwatch.MetricAlarm(stepAlarmName, { namespace: "AWS/ECS", metricName: "CPUUtilization", dimensions: { ClusterName: cluster.clusterName, ServiceName: service.name }, statistic: "Maximum", period: AppProvisioner._stepScaleOutAlarmPeriodSeconds, evaluationPeriods: 1, datapointsToAlarm: 1, threshold: AppProvisioner._stepScaleOutCpuThresholdPercent, comparisonOperator: "GreaterThanOrEqualToThreshold", treatMissingData: "notBreaching", alarmDescription: `${this._name}: hottest task CPU >= ${AppProvisioner._stepScaleOutCpuThresholdPercent}% over one minute; step scale-out`, alarmActions: [stepPolicy.arn], tags: { ...NfraConfig.tags, Name: stepAlarmName } }); } private _isPublicImage(): boolean { const image = this._config.image.trim().toLowerCase(); return AppProvisioner._publicImageRegistries.some(t => image.startsWith(t)); } private async _verifyImageExists(): Promise { if (this._isPublicImage()) return; const [imageRepo, imageTag] = this._config.image.split(":"); const ecrClient = new ECRClient({ region: NfraConfig.ecrAwsRegion }); const command = new DescribeImagesCommand({ registryId: NfraConfig.ecrAwsAccountId, repositoryName: imageRepo, imageIds: [{ imageTag: imageTag }] }); try { const response = await ecrClient.send(command); if (response.imageDetails == null || response.imageDetails.isEmpty) throw new Error(`no image with tag '${imageTag}' in repository '${imageRepo}'`); } catch (error) { const cause = error instanceof Error ? `${error.name}: ${error.message}` : String(error); const message = `image '${this._config.image}' for service '${this._name}' not found in ECR (DescribeImages failed: ${cause})`; console.warn(message); console.error(error); try { await Logger.logWarning(message); await Logger.logError(error as any); } // eslint-disable-next-line no-empty catch { } throw new ArgumentException("image", message); } } private _stringifyContainerDefinitions(containerDefinitions: Record): Pulumi.Output { return Object.entries(containerDefinitions) .map(entry => { return { ...entry[1], name: entry[0] }; }) .map(container => { return Object.entries(container) .map(entry => { const key = JSON.stringify(entry[0]); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (entry[1] === undefined) return null; /** * Lets make some assumptions * Right side is either * 1. primitive => Just use * 2. Applicable => Just use * 3. Array => iterate and check value type and take action * 4. Object => browse entries and take action * * */ // const rightSide = entry[1]; // const isOutput = (input: any): boolean => // { // return input != null // // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition // && (>input).apply != null // && typeof (>input).apply === "function"; // }; // // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition // const value = entry[1] != null && (>entry[1]).apply != null && typeof (>entry[1]).apply === "function" // ? (>entry[1]).apply((val: any) => JSON.stringify(val)) // : JSON.stringify(entry[1]); // return Pulumi.interpolate`${key}:${value}`; return Pulumi.interpolate`${key}:${this._interpolatify(entry[1])}`; }) .where(t => t != null) .reduce((acc, value, index, all) => { acc = Pulumi.interpolate`${acc}${index === 0 ? "" : ","}${value}${index === (all.length - 1) ? "}" : ""}`; // FIXME: adding the trailing '}' here only works because we are guaranteed to not have an empty object return acc; }, Pulumi.interpolate`{`); }) .reduce((acc, value, index, all) => { acc = Pulumi.interpolate`${acc}${index === 0 ? "" : ","}${value}${index === (all.length - 1) ? "]" : ""}`; // FIXME: adding the trailing ']' here only works because we are guaranteed to not have an empty array return acc; }, Pulumi.interpolate`[`) as Pulumi.Output; } private _interpolatify(value: any): Pulumi.Output { if (typeof value === "string") { if (value.trim().startsWith(`"`) && value.trim().endsWith(`"`)) return value as unknown as Pulumi.Output; return `"${value}"` as unknown as Pulumi.Output; } if (typeof value === "number" || typeof value === "boolean") return `${value}` as unknown as Pulumi.Output; const isApplicable = (input: any): boolean => { return input != null // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition && (>input).apply != null && typeof (>input).apply === "function"; }; if (isApplicable(value)) return (value as Pulumi.Output).apply(t => this._interpolatify(t)); if (Array.isArray(value)) { const interpolated = new Array>(); for (const item of value) interpolated.push(this._interpolatify(item)); const interpolatedString = interpolated.reduce((acc, val, index) => { acc = Pulumi.interpolate`${acc}${index === 0 ? "" : ","}${val}`; return acc; }, Pulumi.interpolate`[`); return Pulumi.interpolate`${interpolatedString}]`; } // object const interpolated = new Array>(); for (const entry of Object.entries(value)) interpolated.push(Pulumi.interpolate`"${entry[0]}":${this._interpolatify(entry[1])}`); const interpolatedString = interpolated.reduce((acc, val, index) => { acc = Pulumi.interpolate`${acc}${index === 0 ? "" : ","}${val}`; return acc; }, Pulumi.interpolate`{`); return Pulumi.interpolate`${interpolatedString}}`; } private _createLogConfiguration(): aws.ecs.LogConfiguration { if (this.hasDatadog) { return { logDriver: "awsfirelens", options: { "Name": "datadog", "Host": `http-intake.logs.${this._config.datadogConfig!.ddHost}`, "TLS": "on", "compress": "gzip", // "dd_env": this._appEnv, // this causes error killing log_router container "dd_service": this._name, "dd_source": "stdout", "dd_tags": Object.entries(NfraConfig.tags) .reduce((acc, entry) => acc + `${acc.length > 0 ? "," : ""}${entry[0]}:${entry[1]}`, "") .trim(), "provider": "ecs" }, secretOptions: [{ name: "apikey", valueFrom: this._config.datadogConfig!.apiKey.arn }] } satisfies aws.ecs.LogConfiguration; } else { // return Pulumi.output({ // logDriver: "awsfirelens", // options: { // "Name": "cloudwatch", // "region": NfraConfig.awsRegion, // "log_key": "message", // // "log_group_name": this._name, // // "log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application", // "log_group_name": `/aws/ecs/${this._name}/application`, // "auto_create_group": "true", // // "log_stream_name": this._name, // "log_stream_name": "$(ecs_task_id)", // "retry_limit": "2" // } // }); return { logDriver: "awslogs", options: { "awslogs-group": `/aws/ecs/${this._name}/application`, "awslogs-create-group": "true", "awslogs-region": NfraConfig.awsRegion, "awslogs-stream-prefix": "ecs" } } satisfies aws.ecs.LogConfiguration; } } private _createAwsLogsConfiguration(containerName: string): aws.ecs.LogConfiguration { given(containerName, "containerName").ensureHasValue().ensureIsString(); return { logDriver: "awslogs", options: { "awslogs-group": `/aws/ecs/${this._name}/${containerName}`, "awslogs-create-group": "true", "awslogs-region": NfraConfig.awsRegion, "awslogs-stream-prefix": "ecs" } } satisfies aws.ecs.LogConfiguration; } private _createInstrumentationEnvironmentVariables(): Array { const result: Array = [ { name: "env", value: this._appEnv } ]; if (this.hasDatadog) result.push( { name: "DD_ENV", value: this._appEnv }, { name: "DD_SERVICE", value: this._name }, { name: "DD_VERSION", value: this._version } ); else if (this._config.enableXray) result.push( { name: "enableXrayTracing", value: "true" } ); return result; } private _createDatadogInstrumentationLabels(): { [label: string]: string; } { const labels = { "com.datadoghq.tags.env": this._appEnv, "com.datadoghq.tags.service": this._name, "com.datadoghq.tags.version": this._version, ...this._config.datadogConfig!.additionalInstrumentationLabels ?? {} }; return labels; } private _createInstrumentationContainers( // vnode: aws.appmesh.VirtualNode, isEc2 = false): Record { const containers: Record = { // "log_router": this._createLogRouterContainer(), // "envoy": this._createEnvoyContainer(vnode) }; if (this.hasDatadog) { containers["log_router"] = this._createLogRouterContainer(); containers["datadog-agent"] = this._createDatadogAgentContainer(isEc2); } else if (this._config.enableXray) containers["xray"] = this._createAwsOtelCollectorContainer(); // this._createAwsXrayDaemonContainer(); return containers; } private _createLogRouterContainer(): aws.ecs.ContainerDefinition { return { name: "log_router", // public.ecr.aws/aws-observability/aws-for-fluent-bit:2.34.1.20251112 image: "public.ecr.aws/aws-observability/aws-for-fluent-bit:2.34.1.20251112", essential: true, readonlyRootFilesystem: true, cpu: 10, memoryReservation: 50, firelensConfiguration: { type: "fluentbit", options: { "enable-ecs-log-metadata": "false", "config-file-type": "file", "config-file-value": "/fluent-bit/configs/parse-json.conf" } }, logConfiguration: this._createAwsLogsConfiguration("log_router"), environment: [], mountPoints: [], portMappings: [], user: "0", volumesFrom: [], healthCheck: { "command": [ "CMD", "nc", "-z", "localhost", "8877" ], "retries": 3, "timeout": 10, "interval": 30, "startPeriod": 15 } }; } // private _createEnvoyContainer(vnode: aws.appmesh.VirtualNode): awsx.ecs.Container // { // given(vnode, "vnode").ensureHasValue().ensureIsObject(); // return { // image: "public.ecr.aws/appmesh/aws-appmesh-envoy:v1.24.0.0-prod", // essential: true, // readonlyRootFilesystem: false, // cpu: 300, // memoryReservation: 256, // environment: vnode.arn.apply(vnodeArn => // { // return [ // { name: "APPMESH_RESOURCE_ARN", value: vnodeArn }, // ...this.hasDatadog // ? [ // { name: "ENABLE_ENVOY_DATADOG_TRACING", value: "1" }, // { name: "DD_ENV", value: this._appEnv }, // { name: "DD_SERVICE", value: this._name } // // { name: "DATADOG_TRACER_PORT", value: "8126" }, // // { name: "DATADOG_TRACER_ADDRESS", value: "127.0.0.1" }, // <- service discovery for the datadog agent // ] // : this._config.enableXray // ? [{ name: "ENABLE_ENVOY_XRAY_TRACING", value: "1" }] // : [], // { name: "ENABLE_ENVOY_STATS_TAGS", value: "1" }, // { name: "ENABLE_ENVOY_DOG_STATSD", value: "1" }, // // { name: "ENVOY_LOG_LEVEL", value: "debug" } // { name: "ENVOY_LOG_LEVEL", value: "off" }, // { name: "LISTENER_DRAIN_WAIT_TIME_S", value: "50" } // ].orderBy(t => t.name); // }), // dockerLabels: this.hasDatadog // ? { // "com.datadoghq.ad.logs": `[{"source": "envoy", "service": "${this._name}-envoy"}]`, // "com.datadoghq.tags.env": this._appEnv, // "com.datadoghq.tags.service": `${this._name}-envoy`, // "com.datadoghq.ad.instances": `[{ "stats_url": "http://%%host%%:9901/stats" }]`, // "com.datadoghq.ad.check_names": `["envoy"]`, // "com.datadoghq.ad.init_configs": `[{}]` // } // : undefined, // // logConfiguration: this._createLogConfiguration(`${service}-envoy`), // logConfiguration: this._createAwsLogsConfiguration("envoy"), // dependsOn: this.hasDatadog // ? [{ // containerName: "datadog-agent", // // condition: "START" // condition: "HEALTHY" // }] // : this._config.enableXray // ? [{ // containerName: "xray", // // condition: "START" // condition: "HEALTHY" // }] // : undefined, // mountPoints: [], // // mountPoints: [ // // { // // readOnly: false, // // containerPath: "/tmp", // // sourceVolume: "tmp-envoy" // // }, // // { // // readOnly: false, // // containerPath: "/etc", // // sourceVolume: "etc-envoy" // // } // // ], // portMappings: [ // { // hostPort: 9901, // containerPort: 9901, // protocol: "tcp" // }, // { // hostPort: 15000, // containerPort: 15000, // protocol: "tcp" // }, // { // hostPort: 15001, // containerPort: 15001, // protocol: "tcp" // } // ], // stopTimeout: 60, // user: "1337", // ulimits: [{ // name: "nofile", // hardLimit: 15000, // softLimit: 15000 // }], // volumesFrom: [], // healthCheck: { // "command": [ // "CMD-SHELL", "curl -s http://localhost:9901/server_info | grep state | grep -q LIVE" // ], // "retries": 3, // "timeout": 10, // "interval": 30, // "startPeriod": 15 // } // }; // } private _createDatadogAgentContainer(isEc2 = false): aws.ecs.ContainerDefinition { given(isEc2, "isEc2").ensureHasValue().ensureIsBoolean(); given(this, "this").ensure(t => t.hasDatadog, "datadog config must be provided"); const environment = [ { name: "DD_SITE", value: this._config.datadogConfig!.ddHost }, { name: "DD_APM_NON_LOCAL_TRAFFIC", value: "true" }, // { name: "ECS_FARGATE", value: "true" }, { name: "DD_APM_ENABLED", value: "true" }, { name: "DD_LOGS_ENABLED", value: "true" }, { name: "DD_DOGSTATSD_NON_LOCAL_TRAFFIC", value: "true" }, // { name: "DD_API_KEY", value: this._config.datadogConfig!.apiKey }, // { name: "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", value: "0.0.0.0:4317"} { name: "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT", value: "0.0.0.0:4318" }, { name: "DD_ECS_TASK_COLLECTION_ENABLED", value: "true" } ]; const mountPoints = [ { sourceVolume: "etc-datadog_agent", containerPath: "/etc/datadog-agent", readOnly: false }, ...this.config.datadogConfig?.containerMountPoints?.map(t => { (t).readOnly = true; return t; }) ?? [] ]; if (isEc2) { environment.push({ name: "DD_ECS_COLLECT_RESOURCE_TAGS_EC2", value: "true" }); mountPoints.push( { sourceVolume: "docker_sock", containerPath: "/var/run/docker.sock", readOnly: false }, { sourceVolume: "cgroup", containerPath: "/host/sys/fs/cgroup", readOnly: false }, { sourceVolume: "proc", containerPath: "/host/proc", readOnly: false } ); } else { environment.push({ name: "ECS_FARGATE", value: "true" }); } return { name: "datadog-agent", image: "public.ecr.aws/datadog/agent:7.73.0-rc.6-jmx", essential: true, readonlyRootFilesystem: true, cpu: 30, memoryReservation: 256, // soft limit portMappings: [ { hostPort: 8126, containerPort: 8126, protocol: "tcp" }, { hostPort: 8125, containerPort: 8125, protocol: "udp" }, { hostPort: 4318, containerPort: 4318, protocol: "tcp" } ], environment: environment.orderBy(t => t.name), secrets: [{ name: "DD_API_KEY", valueFrom: this._config.datadogConfig!.apiKey.arn }], // .orderBy(t => t.name) // .map(t => t.arn.apply(u => ({ name: t.name, valueFrom: u }))), dockerLabels: { "com.datadoghq.ad.logs": JSON.stringify(`[{"source": "datadog-agent", "service": "datadog-agent"}]`) }, logConfiguration: this._createAwsLogsConfiguration("datadog-agent"), mountPoints, volumesFrom: [], healthCheck: { "command": [ "CMD-SHELL", "agent health" ], "retries": 3, "timeout": 10, "interval": 30, "startPeriod": 15 } }; } // // @ts-expect-error: not used atm // private _createAwsXrayDaemonContainer(): Container // { // given(this, "this").ensure(t => !t.hasDatadog, "cannot use Xray when datadog config is provided"); // return { // image: "public.ecr.aws/xray/aws-xray-daemon:3.3.5", // essential: true, // readonlyRootFilesystem: false, // cpu: 30, // memoryReservation: 256, // portMappings: [{ // hostPort: 2000, // containerPort: 2000, // protocol: "udp" // }], // environment: [], // logConfiguration: this._createAwsLogsConfiguration("xray"), // mountPoints: [], // volumesFrom: [], // healthCheck: { // "command": [ // "CMD-SHELL", // "timeout 1 /bin/bash -c \" t._config.enableXray!, "xray must be enabled"); return { name: "xray", image: "public.ecr.aws/aws-observability/aws-otel-collector:v0.25.0", essential: true, readonlyRootFilesystem: false, cpu: 30, memoryReservation: 256, command: ["--config=/etc/ecs/ecs-default-config.yaml"], portMappings: [ { hostPort: 2000, containerPort: 2000, protocol: "udp" }, { hostPort: 8125, containerPort: 8125, protocol: "udp" }, { hostPort: 4318, containerPort: 4318, protocol: "tcp" } ], environment: [], logConfiguration: this._createAwsLogsConfiguration("xray"), mountPoints: [], volumesFrom: [], healthCheck: { "command": ["/healthcheck"], "retries": 3, "timeout": 10, "interval": 30, "startPeriod": 15 } }; } } enum CapacityProvider { fargate = "FARGATE", fargateSpot = "FARGATE_SPOT" } /** @internal */ function _defaultRoleAssumeRolePolicy(): aws.iam.PolicyDocument { return { "Version": "2012-10-17", "Statement": [{ "Action": "sts:AssumeRole", "Principal": { "Service": "ecs-tasks.amazonaws.com", }, "Effect": "Allow", "Sid": "", }], }; } /** @internal */ function _createRole( name: string, assumeRolePolicy: string | aws.iam.PolicyDocument, policyArns: Array, opts: Pulumi.ComponentResourceOptions | undefined): aws.iam.Role { return _createRoleAndPolicies(name, assumeRolePolicy, policyArns, opts); } /** @internal */ function _createRoleAndPolicies( name: string, assumeRolePolicy: string | aws.iam.PolicyDocument, policyArns: Array, opts: Pulumi.ComponentResourceOptions | undefined): aws.iam.Role { if (typeof assumeRolePolicy !== "string") { assumeRolePolicy = JSON.stringify(assumeRolePolicy); } const role = new aws.iam.Role(name, { assumeRolePolicy, forceDetachPolicies: true }, opts); const policies: Array = []; for (let i = 0; i < policyArns.length; i++) { const policyArn = policyArns[i]; policies.push(new aws.iam.RolePolicyAttachment( `${name}-${_sha1hash(policyArn)}`, { role, policyArn }, opts)); } return role; } // sha1hash returns a partial SHA1 hash of the input string. /** @internal */ function _sha1hash(s: string): string { const shasum: crypto.Hash = crypto.createHash("sha1"); shasum.update(s); // TODO[pulumi/pulumi#377] Workaround for issue with long names not generating per-deplioyment randomness, leading // to collisions. For now, limit the size of hashes to ensure we generate shorter/ resource names. return shasum.digest("hex").substring(0, 8); }