import * as pulumi from "@pulumi/pulumi"; import * as inputs from "../types/input"; import * as outputs from "../types/output"; /** * Represents a user-visible job which provides the insights for the related data source. * * To get more information about Datascan, see: * * * [API documentation](https://cloud.google.com/dataplex/docs/reference/rest) * * How-to Guides * * [Official Documentation](https://cloud.google.com/dataplex/docs) * * ## Example Usage * * ### Dataplex Datascan Basic Profile * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const basicProfile = new gcp.dataplex.Datascan("basic_profile", { * location: "us-central1", * dataScanId: "dataprofile-basic", * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * dataProfileSpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Full Profile * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const source = new gcp.bigquery.Dataset("source", { * datasetId: "dataplex_dataset", * friendlyName: "test", * description: "This is a test description", * location: "US", * deleteContentsOnDestroy: true, * }); * const fullProfile = new gcp.dataplex.Datascan("full_profile", { * location: "us-central1", * displayName: "Full Datascan Profile", * dataScanId: "dataprofile-full", * description: "Example resource - Full Datascan Profile", * labels: { * author: "billing", * }, * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * schedule: { * cron: "TZ=America/New_York 1 1 * * *", * }, * }, * }, * dataProfileSpec: { * samplingPercent: 80, * rowFilter: "word_count > 10", * includeFields: { * fieldNames: ["word_count"], * }, * excludeFields: { * fieldNames: ["property_type"], * }, * postScanActions: { * bigqueryExport: { * resultsTable: "//bigquery.googleapis.com/projects/my-project-name/datasets/dataplex_dataset/tables/profile_export", * }, * }, * catalogPublishingEnabled: true, * }, * project: "my-project-name", * }, { * dependsOn: [source], * }); * ``` * ### Dataplex Datascan Onetime Profile * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const onetimeProfile = new gcp.dataplex.Datascan("onetime_profile", { * location: "us-central1", * dataScanId: "dataprofile-onetime", * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * oneTime: { * ttlAfterScanCompletion: "120s", * }, * }, * }, * dataProfileSpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Basic Quality * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const basicQuality = new gcp.dataplex.Datascan("basic_quality", { * location: "us-central1", * dataScanId: "dataquality-basic", * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * dataQualitySpec: { * rules: [{ * dimension: "VALIDITY", * name: "rule1", * description: "rule 1 for validity dimension", * tableConditionExpectation: { * sqlExpression: "COUNT(*) > 0", * }, * }], * }, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Full Quality * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const fullQuality = new gcp.dataplex.Datascan("full_quality", { * location: "us-central1", * displayName: "Full Datascan Quality", * dataScanId: "dataquality-full", * description: "Example resource - Full Datascan Quality", * labels: { * author: "billing", * }, * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations", * }, * executionSpec: { * trigger: { * schedule: { * cron: "TZ=America/New_York 1 1 * * *", * }, * }, * field: "modified_date", * }, * dataQualitySpec: { * samplingPercent: 5, * rowFilter: "station_id > 1000", * catalogPublishingEnabled: true, * postScanActions: { * notificationReport: { * recipients: { * emails: ["jane.doe@example.com"], * }, * scoreThresholdTrigger: { * scoreThreshold: 86, * }, * }, * }, * rules: [ * { * column: "address", * dimension: "VALIDITY", * threshold: 0.99, * nonNullExpectation: {}, * }, * { * column: "council_district", * dimension: "VALIDITY", * ignoreNull: true, * threshold: 0.9, * rangeExpectation: { * minValue: "1", * maxValue: "10", * strictMinEnabled: true, * strictMaxEnabled: false, * }, * }, * { * column: "power_type", * dimension: "VALIDITY", * ignoreNull: false, * regexExpectation: { * regex: ".*solar.*", * }, * }, * { * column: "property_type", * dimension: "VALIDITY", * ignoreNull: false, * setExpectation: { * values: [ * "sidewalk", * "parkland", * ], * }, * }, * { * column: "address", * dimension: "UNIQUENESS", * uniquenessExpectation: {}, * }, * { * column: "number_of_docks", * dimension: "VALIDITY", * statisticRangeExpectation: { * statistic: "MEAN", * minValue: "5", * maxValue: "15", * strictMinEnabled: true, * strictMaxEnabled: true, * }, * }, * { * column: "footprint_length", * dimension: "VALIDITY", * rowConditionExpectation: { * sqlExpression: "footprint_length > 0 AND footprint_length <= 10", * }, * }, * { * dimension: "VALIDITY", * tableConditionExpectation: { * sqlExpression: "COUNT(*) > 0", * }, * }, * { * dimension: "VALIDITY", * sqlAssertion: { * sqlStatement: "select * from bigquery-public-data.austin_bikeshare.bikeshare_stations where station_id is null", * }, * }, * ], * }, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Onetime Quality * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const onetimeQuality = new gcp.dataplex.Datascan("onetime_quality", { * location: "us-central1", * dataScanId: "dataquality-onetime", * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * oneTime: { * ttlAfterScanCompletion: "120s", * }, * }, * }, * dataQualitySpec: { * rules: [{ * dimension: "VALIDITY", * name: "rule1", * description: "rule 1 for validity dimension", * tableConditionExpectation: { * sqlExpression: "COUNT(*) > 0", * }, * }], * }, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Basic Discovery * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfTestBucket = new gcp.storage.Bucket("tf_test_bucket", { * name: "tf-test-bucket-name-_16511", * location: "us-west1", * uniformBucketLevelAccess: true, * }); * const basicDiscovery = new gcp.dataplex.Datascan("basic_discovery", { * location: "us-central1", * dataScanId: "datadiscovery-basic", * data: { * resource: pulumi.interpolate`//storage.googleapis.com/projects/${tfTestBucket.project}/buckets/${tfTestBucket.name}`, * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * dataDiscoverySpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Full Discovery * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfTestBucket = new gcp.storage.Bucket("tf_test_bucket", { * name: "tf-test-bucket-name-_8493", * location: "us-west1", * uniformBucketLevelAccess: true, * }); * const tfTestConnection = new gcp.bigquery.Connection("tf_test_connection", { * connectionId: "tf-test-connection-_9106", * location: "us-central1", * friendlyName: "tf-test-connection-_27169", * description: "a bigquery connection for tf test", * cloudResource: {}, * }); * const fullDiscovery = new gcp.dataplex.Datascan("full_discovery", { * location: "us-central1", * displayName: "Full Datascan Discovery", * dataScanId: "datadiscovery-full", * description: "Example resource - Full Datascan Discovery", * labels: { * author: "billing", * }, * data: { * resource: pulumi.interpolate`//storage.googleapis.com/projects/${tfTestBucket.project}/buckets/${tfTestBucket.name}`, * }, * executionSpec: { * trigger: { * schedule: { * cron: "TZ=America/New_York 1 1 * * *", * }, * }, * }, * dataDiscoverySpec: { * bigqueryPublishingConfig: { * tableType: "BIGLAKE", * connection: pulumi.all([tfTestConnection.project, tfTestConnection.location, tfTestConnection.connectionId]).apply(([project, location, connectionId]) => `projects/${project}/locations/${location}/connections/${connectionId}`), * location: tfTestBucket.location, * project: pulumi.interpolate`projects/${tfTestBucket.project}`, * }, * storageConfig: { * includePatterns: [ * "ai*", * "ml*", * ], * excludePatterns: [ * "doc*", * "gen*", * ], * csvOptions: { * headerRows: 5, * delimiter: ",", * encoding: "UTF-8", * typeInferenceDisabled: false, * quote: "'", * }, * jsonOptions: { * encoding: "UTF-8", * typeInferenceDisabled: false, * }, * }, * }, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Onetime Discovery * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfTestBucket = new gcp.storage.Bucket("tf_test_bucket", { * name: "tf-test-bucket-name-_75223", * location: "us-west1", * uniformBucketLevelAccess: true, * }); * const onetimeDiscovery = new gcp.dataplex.Datascan("onetime_discovery", { * location: "us-central1", * dataScanId: "datadiscovery-onetime", * data: { * resource: pulumi.interpolate`//storage.googleapis.com/projects/${tfTestBucket.project}/buckets/${tfTestBucket.name}`, * }, * executionSpec: { * trigger: { * oneTime: { * ttlAfterScanCompletion: "120s", * }, * }, * }, * dataDiscoverySpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Documentation * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfDataplexTestDataset = new gcp.bigquery.Dataset("tf_dataplex_test_dataset", { * datasetId: "tf_dataplex_test_dataset_id__41819", * defaultTableExpirationMs: 3600000, * }); * const tfDataplexTestTable = new gcp.bigquery.Table("tf_dataplex_test_table", { * datasetId: tfDataplexTestDataset.datasetId, * tableId: "tf_dataplex_test_table_id__75092", * deletionProtection: false, * schema: ` [ * { * \\"name\\": \\"name\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\" * }, * { * \\"name\\": \\"station_id\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The id of the bike station\\" * }, * { * \\"name\\": \\"address\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The address of the bike station\\" * }, * { * \\"name\\": \\"power_type\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The powert type of the bike station\\" * }, * { * \\"name\\": \\"property_type\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The type of the property\\" * }, * { * \\"name\\": \\"number_of_docks\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The number of docks the property have\\" * }, * { * \\"name\\": \\"footprint_length\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The footpring lenght of the property\\" * }, * { * \\"name\\": \\"council_district\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The council district the property is in\\" * } * ] * `, * }); * const documentation = new gcp.dataplex.Datascan("documentation", { * location: "us-central1", * dataScanId: "datadocumentation", * data: { * resource: pulumi.interpolate`//bigquery.googleapis.com/projects/my-project-name/datasets/${tfDataplexTestDataset.datasetId}/tables/${tfDataplexTestTable.tableId}`, * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * dataDocumentationSpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Onetime Documentation * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfDataplexTestDataset = new gcp.bigquery.Dataset("tf_dataplex_test_dataset", { * datasetId: "tf_dataplex_test_dataset_id__2605", * defaultTableExpirationMs: 3600000, * }); * const tfDataplexTestTable = new gcp.bigquery.Table("tf_dataplex_test_table", { * datasetId: tfDataplexTestDataset.datasetId, * tableId: "tf_dataplex_test_table_id__34535", * deletionProtection: false, * schema: ` [ * { * \\"name\\": \\"name\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\" * }, * { * \\"name\\": \\"station_id\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The id of the bike station\\" * }, * { * \\"name\\": \\"address\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The address of the bike station\\" * }, * { * \\"name\\": \\"power_type\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The powert type of the bike station\\" * }, * { * \\"name\\": \\"property_type\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The type of the property\\" * }, * { * \\"name\\": \\"number_of_docks\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The number of docks the property have\\" * }, * { * \\"name\\": \\"footprint_length\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The footpring lenght of the property\\" * }, * { * \\"name\\": \\"council_district\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The council district the property is in\\" * } * ] * `, * }); * const onetimeDocumentation = new gcp.dataplex.Datascan("onetime_documentation", { * location: "us-central1", * dataScanId: "datadocumentation-onetime", * data: { * resource: pulumi.interpolate`//bigquery.googleapis.com/projects/my-project-name/datasets/${tfDataplexTestDataset.datasetId}/tables/${tfDataplexTestTable.tableId}`, * }, * executionSpec: { * trigger: { * oneTime: { * ttlAfterScanCompletion: "120s", * }, * }, * }, * dataDocumentationSpec: {}, * project: "my-project-name", * }); * ``` * * ## Import * * Datascan can be imported using any of these accepted formats: * * * `projects/{{project}}/locations/{{location}}/dataScans/{{data_scan_id}}` * * `{{project}}/{{location}}/{{data_scan_id}}` * * `{{location}}/{{data_scan_id}}` * * `{{data_scan_id}}` * * When using the `pulumi import` command, Datascan can be imported using one of the formats above. For example: * * ```sh * $ pulumi import gcp:dataplex/datascan:Datascan default projects/{{project}}/locations/{{location}}/dataScans/{{data_scan_id}} * $ pulumi import gcp:dataplex/datascan:Datascan default {{project}}/{{location}}/{{data_scan_id}} * $ pulumi import gcp:dataplex/datascan:Datascan default {{location}}/{{data_scan_id}} * $ pulumi import gcp:dataplex/datascan:Datascan default {{data_scan_id}} * ``` */ export declare class Datascan extends pulumi.CustomResource { /** * Get an existing Datascan resource's state with the given name, ID, and optional extra * properties used to qualify the lookup. * * @param name The _unique_ name of the resulting resource. * @param id The _unique_ provider ID of the resource to lookup. * @param state Any extra arguments used during the lookup. * @param opts Optional settings to control the behavior of the CustomResource. */ static get(name: string, id: pulumi.Input, state?: DatascanState, opts?: pulumi.CustomResourceOptions): Datascan; /** * Returns true if the given object is an instance of Datascan. This is designed to work even * when multiple copies of the Pulumi SDK have been loaded into the same process. */ static isInstance(obj: any): obj is Datascan; /** * The time when the scan was created. */ readonly createTime: pulumi.Output; /** * The data source for DataScan. * Structure is documented below. */ readonly data: pulumi.Output; /** * DataDiscoveryScan related setting. * Structure is documented below. */ readonly dataDiscoverySpec: pulumi.Output; /** * DataDocumentationScan related setting. */ readonly dataDocumentationSpec: pulumi.Output; /** * DataProfileScan related setting. * Structure is documented below. */ readonly dataProfileSpec: pulumi.Output; /** * DataQualityScan related setting. * Structure is documented below. */ readonly dataQualitySpec: pulumi.Output; /** * DataScan identifier. Must contain only lowercase letters, numbers and hyphens. Must start with a letter. Must end with a number or a letter. */ readonly dataScanId: pulumi.Output; /** * Description of the scan. */ readonly description: pulumi.Output; /** * User friendly display name. */ readonly displayName: pulumi.Output; /** * All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services. */ readonly effectiveLabels: pulumi.Output<{ [key: string]: string; }>; /** * DataScan execution settings. * Structure is documented below. */ readonly executionSpec: pulumi.Output; /** * Status of the data scan execution. * Structure is documented below. */ readonly executionStatuses: pulumi.Output; /** * User-defined labels for the scan. A list of key->value pairs. * * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ readonly labels: pulumi.Output<{ [key: string]: string; } | undefined>; /** * The location where the data scan should reside. */ readonly location: pulumi.Output; /** * The relative resource name of the scan, of the form: projects/{project}/locations/{locationId}/dataScans/{datascan_id}, where project refers to a projectId or projectNumber and locationId refers to a GCP region. */ readonly name: pulumi.Output; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ readonly project: pulumi.Output; /** * The combination of labels configured directly on the resource * and default labels configured on the provider. */ readonly pulumiLabels: pulumi.Output<{ [key: string]: string; }>; /** * Current state of the DataScan. */ readonly state: pulumi.Output; /** * The type of DataScan. */ readonly type: pulumi.Output; /** * System generated globally unique ID for the scan. This ID will be different if the scan is deleted and re-created with the same name. */ readonly uid: pulumi.Output; /** * The time when the scan was last updated. */ readonly updateTime: pulumi.Output; /** * Create a Datascan resource with the given unique name, arguments, and options. * * @param name The _unique_ name of the resource. * @param args The arguments to use to populate this resource's properties. * @param opts A bag of options that control this resource's behavior. */ constructor(name: string, args: DatascanArgs, opts?: pulumi.CustomResourceOptions); } /** * Input properties used for looking up and filtering Datascan resources. */ export interface DatascanState { /** * The time when the scan was created. */ createTime?: pulumi.Input; /** * The data source for DataScan. * Structure is documented below. */ data?: pulumi.Input; /** * DataDiscoveryScan related setting. * Structure is documented below. */ dataDiscoverySpec?: pulumi.Input; /** * DataDocumentationScan related setting. */ dataDocumentationSpec?: pulumi.Input; /** * DataProfileScan related setting. * Structure is documented below. */ dataProfileSpec?: pulumi.Input; /** * DataQualityScan related setting. * Structure is documented below. */ dataQualitySpec?: pulumi.Input; /** * DataScan identifier. Must contain only lowercase letters, numbers and hyphens. Must start with a letter. Must end with a number or a letter. */ dataScanId?: pulumi.Input; /** * Description of the scan. */ description?: pulumi.Input; /** * User friendly display name. */ displayName?: pulumi.Input; /** * All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services. */ effectiveLabels?: pulumi.Input<{ [key: string]: pulumi.Input; }>; /** * DataScan execution settings. * Structure is documented below. */ executionSpec?: pulumi.Input; /** * Status of the data scan execution. * Structure is documented below. */ executionStatuses?: pulumi.Input[]>; /** * User-defined labels for the scan. A list of key->value pairs. * * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ labels?: pulumi.Input<{ [key: string]: pulumi.Input; }>; /** * The location where the data scan should reside. */ location?: pulumi.Input; /** * The relative resource name of the scan, of the form: projects/{project}/locations/{locationId}/dataScans/{datascan_id}, where project refers to a projectId or projectNumber and locationId refers to a GCP region. */ name?: pulumi.Input; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ project?: pulumi.Input; /** * The combination of labels configured directly on the resource * and default labels configured on the provider. */ pulumiLabels?: pulumi.Input<{ [key: string]: pulumi.Input; }>; /** * Current state of the DataScan. */ state?: pulumi.Input; /** * The type of DataScan. */ type?: pulumi.Input; /** * System generated globally unique ID for the scan. This ID will be different if the scan is deleted and re-created with the same name. */ uid?: pulumi.Input; /** * The time when the scan was last updated. */ updateTime?: pulumi.Input; } /** * The set of arguments for constructing a Datascan resource. */ export interface DatascanArgs { /** * The data source for DataScan. * Structure is documented below. */ data: pulumi.Input; /** * DataDiscoveryScan related setting. * Structure is documented below. */ dataDiscoverySpec?: pulumi.Input; /** * DataDocumentationScan related setting. */ dataDocumentationSpec?: pulumi.Input; /** * DataProfileScan related setting. * Structure is documented below. */ dataProfileSpec?: pulumi.Input; /** * DataQualityScan related setting. * Structure is documented below. */ dataQualitySpec?: pulumi.Input; /** * DataScan identifier. Must contain only lowercase letters, numbers and hyphens. Must start with a letter. Must end with a number or a letter. */ dataScanId: pulumi.Input; /** * Description of the scan. */ description?: pulumi.Input; /** * User friendly display name. */ displayName?: pulumi.Input; /** * DataScan execution settings. * Structure is documented below. */ executionSpec: pulumi.Input; /** * User-defined labels for the scan. A list of key->value pairs. * * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ labels?: pulumi.Input<{ [key: string]: pulumi.Input; }>; /** * The location where the data scan should reside. */ location: pulumi.Input; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ project?: pulumi.Input; }