import type * as constructs from 'constructs'; import * as cloudwatch from '../../../aws-cloudwatch'; import * as events from '../../../aws-events'; import * as iam from '../../../aws-iam'; import type * as logs from '../../../aws-logs'; import * as cdk from '../../../core'; import type { Code } from '../code'; import type { IConnection } from '../connection'; import type { MetricType, GlueVersion } from '../constants'; import { JobState } from '../constants'; import type { IJobRef, JobReference } from '../glue.generated'; import type { ISecurityConfiguration } from '../security-configuration'; /** * Interface representing a new or an imported Glue Job */ export interface IJob extends cdk.IResource, iam.IGrantable, IJobRef { /** * The name of the job. * @attribute */ readonly jobName: string; /** * The ARN of the job. * @attribute */ readonly jobArn: string; /** * Defines a CloudWatch event rule triggered when something happens with this job. * * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types */ onEvent(id: string, options?: events.OnEventOptions): events.Rule; /** * Defines a CloudWatch event rule triggered when this job moves to the SUCCEEDED state. * * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types */ onSuccess(id: string, options?: events.OnEventOptions): events.Rule; /** * Defines a CloudWatch event rule triggered when this job moves to the FAILED state. * * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types */ onFailure(id: string, options?: events.OnEventOptions): events.Rule; /** * Defines a CloudWatch event rule triggered when this job moves to the TIMEOUT state. * * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types */ onTimeout(id: string, options?: events.OnEventOptions): events.Rule; /** * Create a CloudWatch metric. * * @param metricName name of the metric typically prefixed with `glue.driver.`, `glue..` or `glue.ALL.`. * @param type the metric type. * @param props metric options. * * @see https://docs.aws.amazon.com/glue/latest/dg/monitoring-awsglue-with-cloudwatch-metrics.html */ metric(metricName: string, type: MetricType, props?: cloudwatch.MetricOptions): cloudwatch.Metric; /** * Create a CloudWatch Metric indicating job success. */ metricSuccess(props?: cloudwatch.MetricOptions): cloudwatch.Metric; /** * Create a CloudWatch Metric indicating job failure. */ metricFailure(props?: cloudwatch.MetricOptions): cloudwatch.Metric; /** * Create a CloudWatch Metric indicating job timeout. */ metricTimeout(props?: cloudwatch.MetricOptions): cloudwatch.Metric; } /** * Properties for enabling Continuous Logging for Glue Jobs. * * @see https://docs.aws.amazon.com/glue/latest/dg/monitor-continuous-logging-enable.html * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ export interface ContinuousLoggingProps { /** * Enable continuous logging. */ readonly enabled: boolean; /** * Specify a custom CloudWatch log group name. * * @default - a log group is created with name `/aws-glue/jobs/logs-v2/`. */ readonly logGroup?: logs.ILogGroup; /** * Specify a custom CloudWatch log stream prefix. * * @default - the job run ID. */ readonly logStreamPrefix?: string; /** * Filter out non-useful Apache Spark driver/executor and Apache Hadoop YARN heartbeat log messages. * * @default true */ readonly quiet?: boolean; /** * Apply the provided conversion pattern. * * This is a Log4j Conversion Pattern to customize driver and executor logs. * * @default `%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n` */ readonly conversionPattern?: string; } /** * A base class is needed to be able to import existing Jobs into a CDK app to * reference as part of a larger stack or construct. JobBase has the subset * of attributes required to identify and reference an existing Glue Job, * as well as some CloudWatch metric convenience functions to configure an * event-driven flow using the job. */ export declare abstract class JobBase extends cdk.Resource implements IJob { abstract readonly jobArn: string; abstract readonly jobName: string; abstract readonly grantPrincipal: iam.IPrincipal; /** * A reference to this Job resource, for use with the generated L1 ref interface. */ get jobRef(): JobReference; /** * Create a CloudWatch Event Rule for this Glue Job when it's in a given state * * @param id construct id * @param options event options. Note that some values are overridden if provided, these are * - eventPattern.source = ['aws.glue'] * - eventPattern.detailType = ['Glue Job State Change', 'Glue Job Run Status'] * - eventPattern.detail.jobName = [this.jobName] * * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types */ onEvent(id: string, options?: events.OnEventOptions): events.Rule; /** * Create a CloudWatch Event Rule for the transition into the input jobState. * * @param id construct id. * @param jobState the job state. * @param options optional event options. */ protected onStateChange(id: string, jobState: JobState, options?: events.OnEventOptions): events.Rule; /** * Create a CloudWatch Event Rule matching JobState.SUCCEEDED. * * @param id construct id. * @param options optional event options. default is {}. */ onSuccess(id: string, options?: events.OnEventOptions): events.Rule; /** * Return a CloudWatch Event Rule matching FAILED state. * * @param id construct id. * @param options optional event options. default is {}. */ onFailure(id: string, options?: events.OnEventOptions): events.Rule; /** * Return a CloudWatch Event Rule matching TIMEOUT state. * * @param id construct id. * @param options optional event options. default is {}. */ onTimeout(id: string, options?: events.OnEventOptions): events.Rule; /** * Create a CloudWatch metric. * * @param metricName name of the metric typically prefixed with `glue.driver.`, `glue..` or `glue.ALL.`. * @param type the metric type. * @param props metric options. * * @see https://docs.aws.amazon.com/glue/latest/dg/monitoring-awsglue-with-cloudwatch-metrics.html */ metric(metricName: string, type: MetricType, props?: cloudwatch.MetricOptions): cloudwatch.Metric; /** * Return a CloudWatch Metric indicating job success. * * This metric is based on the Rule returned by no-args onSuccess() call. */ metricSuccess(props?: cloudwatch.MetricOptions): cloudwatch.Metric; /** * Return a CloudWatch Metric indicating job failure. * * This metric is based on the Rule returned by no-args onFailure() call. */ metricFailure(props?: cloudwatch.MetricOptions): cloudwatch.Metric; /** * Return a CloudWatch Metric indicating job timeout. * * This metric is based on the Rule returned by no-args onTimeout() call. */ metricTimeout(props?: cloudwatch.MetricOptions): cloudwatch.Metric; /** * Creates or retrieves a singleton event rule for the input job state for use with the metric JobState methods. * * @param id construct id. * @param jobState the job state. */ private metricJobStateRule; /** * Returns the job arn */ protected buildJobArn(scope: constructs.Construct, jobName: string): string; } /** * A subset of Job attributes are required for importing an existing job * into a CDK project. This is only used when using fromJobAttributes * to identify and reference the existing job. */ export interface JobAttributes { /** * The name of the job. */ readonly jobName: string; /** * The IAM role assumed by Glue to run this job. * * @default - undefined */ readonly role?: iam.IRole; } /** * JobProps will be used to create new Glue Jobs using this L2 Construct. */ export interface JobProps { /** * Script Code Location (required) * Script to run when the Glue job executes. Can be uploaded * from the local directory structure using fromAsset * or referenced via S3 location using fromBucket */ readonly script: Code; /** * IAM Role (required) * IAM Role to use for Glue job execution * Must be specified by the developer because the L2 doesn't have visibility * into the actions the script(s) takes during the job execution * The role must trust the Glue service principal (glue.amazonaws.com) * and be granted sufficient permissions. * * @see https://docs.aws.amazon.com/glue/latest/dg/getting-started-access.html */ readonly role: iam.IRole; /** * Name of the Glue job (optional) * Developer-specified name of the Glue job * * @default - a name is automatically generated */ readonly jobName?: string; /** * Description (optional) * Developer-specified description of the Glue job * * @default - no value */ readonly description?: string; /** * Max Concurrent Runs (optional) * The maximum number of runs this Glue job can concurrently run * * An error is returned when this threshold is reached. The maximum value * you can specify is controlled by a service limit. * * @default 1 */ readonly maxConcurrentRuns?: number; /** * Default Arguments (optional) * The default arguments for every run of this Glue job, * specified as name-value pairs. * * This map is the escape hatch for Glue job arguments that this construct does not model. It * MUST NOT be used to set arguments that already have a dedicated prop — configure those through * the corresponding prop instead (`continuousLogging`, `enableMetrics`, * `enableObservabilityMetrics`, `sparkUI`, `className`, `extraJars`, `extraJarsFirst`, * `extraPythonFiles`, `extraFiles`). Passing a construct-managed argument (e.g. * `--enable-continuous-cloudwatch-log`, `--enable-metrics`, `--enable-spark-ui`, * `--job-language`) or a Glue-reserved argument (`--debug`, `--mode`, `--JOB_NAME`, `--endpoint`) * here throws at synthesis time, so there is exactly one way to express each intent. * * Also note that these are emitted verbatim into the CloudFormation template, so avoid * placing secrets here in plaintext. Pass secrets to the job at runtime * through AWS Secrets Manager instead. A synthesis-time warning is emitted * when an argument key looks like a credential and holds a plaintext literal. * * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html * for a list of reserved parameters * @default - no arguments */ readonly defaultArguments?: { [key: string]: string; }; /** * Connections (optional) * List of connections to use for this Glue job * Connections are used to connect to other AWS Service or resources within a VPC. * * @default [] - no connections are added to the job */ readonly connections?: IConnection[]; /** * Max Retries (optional) * Maximum number of retry attempts Glue performs if the job fails * * @default 0 */ readonly maxRetries?: number; /** * Timeout (optional) * The maximum time that a job run can consume resources before it is * terminated and enters TIMEOUT status. Specified in minutes. * * @default - no value set; Glue applies its service default (2880 minutes for non-streaming jobs) * */ readonly timeout?: cdk.Duration; /** * Security Configuration (optional) * Defines the encryption options for the Glue job * * @default - no security configuration. */ readonly securityConfiguration?: ISecurityConfiguration; /** * Tags (optional) * A list of key:value pairs of tags to apply to this Glue job resources * * @default {} - no tags */ readonly tags?: { [key: string]: string; }; /** * Glue Version * The version of Glue to use to execute this job * * @default - determined by the job type: 4.0 for ETL and Streaming, 5.0 for Flex, 3.0 for Python Shell */ readonly glueVersion?: GlueVersion; /** * Enables continuous logging with the specified props. * * @default - continuous logging is enabled. * * @see https://docs.aws.amazon.com/glue/latest/dg/monitor-continuous-logging-enable.html * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html **/ readonly continuousLogging?: ContinuousLoggingProps; } /** * A Glue Job. * @resource AWS::Glue::Job */ export declare abstract class Job extends JobBase { /** * Identifies an existing Glue Job from a subset of attributes that can * be referenced from within another Stack or Construct. * * @param scope The scope creating construct (usually `this`) * @param id The construct's id. * @param attrs Attributes for the Glue Job we want to import */ static fromJobAttributes(scope: constructs.Construct, id: string, attrs: JobAttributes): IJob; /** * Argument keys that Glue reserves for its own use and that a caller must never set through * `defaultArguments`. Unlike construct-managed arguments (which are derived from what each job * class emits), these are owned by the Glue service itself and are reserved for every job type. * * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ private static readonly GLUE_RESERVED_ARGUMENTS; /** * The IAM role Glue assumes to run this job. */ abstract readonly role: iam.IRole; /** * The construct-managed job arguments that are actually emitted for the current configuration, * accumulated via {@link setManagedArgument}. Passed through to `CfnJob.defaultArguments` after * being merged with the user-supplied `defaultArguments` by {@link mergeDefaultArguments}. */ private readonly _managedArguments; /** * Every argument key this construct manages, accumulated via {@link setManagedArgument} — whether * or not the current configuration emits a value for it. This is the reserved set that a * user-supplied `defaultArguments` must not collide with. Registering a key even when its value * is `undefined` (feature disabled or unset) is what keeps `defaultArguments` from silently * re-enabling it. */ private readonly _managedArgumentKeys; /** * Declare `key` as construct-managed and, when `value` is defined, emit it into the job's * arguments. * * This is the single sink for every argument a job construct derives from its typed props (or * from the job class itself). Call it once per managed key, passing `undefined` as the value when * the corresponding feature is turned off or unset — the key is still reserved from * `defaultArguments` either way, so a disabled feature cannot be re-enabled through the escape * hatch. There is deliberately no other way for a subclass to emit a managed argument, so the * reserved set can never drift from what is emitted. * * @param key the Glue argument key, e.g. `--enable-metrics` * @param value the value to emit, or `undefined` to reserve the key without emitting it */ protected setManagedArgument(key: string, value?: string): void; /** * Merge the customer-supplied `defaultArguments` with the arguments this construct manages. * * The construct owns every argument it emits — whether the value comes from a dedicated typed * prop (e.g. `continuousLogging`, `enableMetrics`, `sparkUI`) or from the job class itself * (e.g. `--job-language`). Those arguments, plus the arguments Glue reserves for its own use, * MUST be configured through their dedicated props rather than the untyped `defaultArguments` * map, so there is exactly one way to express each intent. Passing such a key through * `defaultArguments` therefore throws instead of silently winning or being silently dropped. * * A managed key whose supplied value is identical to the construct's value is not contradictory, * so it is allowed rather than rejected (auto-correcting config is preferred over errors). * Glue-reserved keys are never emitted by the construct, so there is no value to reconcile and * they always throw. * * The reserved set is `_managedArgumentKeys` — every key the construct declared through * {@link setManagedArgument}, whether or not a value was emitted for it. It is deliberately NOT * derived from the keys that carry a value: a typed prop that turns a feature *off* (e.g. * `enableMetrics: false`) emits no value but still reserves its key, so `defaultArguments` cannot * silently re-enable it. * * Conflict detection relies on string equality of the argument keys, which cannot see through * unresolved tokens (e.g. a key produced by `CfnJson` that only resolves at deploy time). If a * key is a token, the check is skipped for that key and a synthesis-time warning is emitted, so * the (rare) case where a token key resolves to a managed argument at deploy time — in which the * construct-managed value would silently take precedence — is surfaced rather than hidden. * * @param defaultArguments the caller-supplied escape-hatch arguments, if any * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ protected mergeDefaultArguments(defaultArguments?: { [key: string]: string; }): { [key: string]: string; }; /** * Register (and, when enabled, emit) the continuous-logging arguments this job manages. * * All five continuous-logging keys are reserved on every job type regardless of configuration: * they are always registered through {@link setManagedArgument}, and carry a value only when * logging is enabled. This keeps `defaultArguments` from re-enabling logging a user turned off. * * @param role The IAM role to grant write access to a custom log group, if one is provided * @param props The properties for continuous logging configuration * @param securityConfiguration The security configuration attached to the job, if any */ protected setupContinuousLogging(role: iam.IRole, props: ContinuousLoggingProps | undefined, securityConfiguration?: ISecurityConfiguration): void; protected codeS3ObjectUrl(code: Code): string; }