/** * @copyright Cube Dev, Inc. * @license Apache-2.0 * @fileoverview The `DatabricksDriver` and related types declaration. */ import { LoggerFn } from '@cubejs-backend/shared'; import { DatabaseStructure, DriverCapabilities, GenericDataBaseType, QueryColumnsResult, QueryOptions, QuerySchemasResult, QueryTablesResult, TableColumn, UnloadOptions } from '@cubejs-backend/base-driver'; import { EscapeDialect, JDBCDriver, JDBCDriverConfiguration } from '@cubejs-backend/jdbc-driver'; import { DatabricksQuery } from './DatabricksQuery'; export type DatabricksDriverConfiguration = JDBCDriverConfiguration & { /** * Driver read-only mode flag. */ readOnly?: boolean; /** * Export bucket type. */ bucketType?: string; /** * Export bucket path. */ exportBucket?: string; /** * Export bucket DBFS mount directory. */ exportBucketMountDir?: string; /** * Poll interval. */ pollInterval?: number; /** * The export bucket CSV file escape symbol. */ exportBucketCsvEscapeSymbol?: string; /** * Export bucket AWS account key. */ awsKey?: string; /** * Export bucket AWS account secret. */ awsSecret?: string; /** * Export bucket AWS account region. */ awsRegion?: string; /** * Export bucket Azure account key. */ azureKey?: string; /** * Databricks catalog name. * https://www.databricks.com/product/unity-catalog */ catalog?: string; /** * Databricks security token (PWD). */ token?: string; /** * Databricks OAuth Client ID. */ oauthClientId?: string; /** * Databricks OAuth Client Secret. */ oauthClientSecret?: string; /** * Azure tenant Id */ azureTenantId?: string; /** * Azure service principal client Id */ azureClientId?: string; /** * Azure service principal client secret */ azureClientSecret?: string; /** * GCS credentials JSON content */ gcsCredentials?: string; }; type ColumnInfo = { name: any; type: GenericDataBaseType; }; export type ParsedConnectionProperties = { host: string; warehouseId: string; }; /** * Databricks driver class. */ export declare class DatabricksDriver extends JDBCDriver { /** * Show warning message flag. */ private readonly showSparkProtocolWarn; /** * Driver Configuration. */ protected readonly config: DatabricksDriverConfiguration; private readonly parsedConnectionProperties; private accessToken; private accessTokenExpires; static dialectClass(): typeof DatabricksQuery; /** * Returns default concurrency value. */ static getDefaultConcurrency(): number; /** * Class constructor. */ constructor(conf?: Partial & { /** * Data source name. */ dataSource?: string; /** * Whether this driver is used for pre-aggregations. */ preAggregations?: boolean; /** * Max pool size value for the [cube]<-->[db] pool. */ maxPoolSize?: number; /** * Time to wait for a response from a connection after validation * request before determining it as not valid. Default - 10000 ms. */ testConnectionTimeout?: number; }); readOnly(): boolean; capabilities(): DriverCapabilities; setLogger(logger: LoggerFn): void; private fetchAccessToken; private getValidAccessToken; testConnection(): Promise; loadPreAggregationIntoTable(preAggregationTableName: string, loadSql: string, params: unknown[], _options: any): Promise; query(query: string, values: unknown[]): Promise; /** * Returns pre-aggregation schema name. */ protected getPreAggrSchemaName(): string; dropTable(tableName: string, options?: QueryOptions): Promise; private showDeprecations; protected getCustomClassPath(): Promise; /** * Execute create schema query. */ createSchemaIfNotExists(schemaName: string): Promise; /** * Returns the list of the tables for the specified schema. */ getTablesQuery(schemaName: string): Promise<{ 'table_name': string; }[]>; /** * Returns tables meta data object. */ tablesSchema(): Promise; /** * Returns list of accessible tables. */ private getTables; getSchemas(): Promise; getTablesForSpecificSchemas(schemas: QuerySchemasResult[]): Promise; getColumnsForSpecificTables(tables: QueryTablesResult[]): Promise; /** * Returns table columns types. */ tableColumnTypes(table: string): Promise; protected escapeDialect(): EscapeDialect; /** * Returns query columns types. */ queryColumnTypes(sql: string, params?: unknown[]): Promise; /** * Returns schema full name. */ private getSchemaFullName; /** * Returns quoted string. */ protected quoteIdentifier(identifier: string): string; /** * Returns the JS type by the Databricks type. */ protected toGenericType(columnType: string, precision?: number | null, scale?: number | null): GenericDataBaseType; /** * Determines whether export bucket feature is configured or not. */ isUnloadSupported(): Promise; /** * Returns to the Cubestore an object with links to unloaded to an * export bucket data. */ unload(tableName: string, options: UnloadOptions): Promise<{ exportBucketCsvEscapeSymbol: string | undefined; csvFile: string[]; types: ColumnInfo[]; csvNoHeader: boolean; }>; /** * Unload data from a SQL query to an export bucket. */ private unloadWithSql; /** * Unload data from a temp table to an export bucket. */ private unloadWithTable; /** * Returns an array of signed URLs of unloaded csv files. */ private getCsvFiles; protected generateTableColumnsForExport(columns: ColumnInfo[]): string; /** * Saves specified query to the configured bucket. This requires Databricks * cluster to be configured. * * For Azure blob storage you need to configure account access key in * Cluster -> Configuration -> Advanced options * https://docs.databricks.com/data/data-sources/azure/azure-storage.html#access-azure-blob-storage-directly * * `fs.azure.account.key..blob.core.windows.net ` * * For S3 bucket storage you need to configure AWS access key and secret in * Cluster -> Configuration -> Advanced options * https://docs.databricks.com/data/data-sources/aws/amazon-s3.html#access-s3-buckets-directly * * `fs.s3a.access.key ` * `fs.s3a.secret.key ` * * For Google cloud storage you can configure storage credentials and create an external location to access it * or configure account service key (legacy) * https://docs.databricks.com/gcp/en/connect/unity-catalog/cloud-storage/storage-credentials * https://docs.databricks.com/gcp/en/connect/unity-catalog/cloud-storage/external-locations * https://docs.databricks.com/aws/en/connect/storage/gcs */ private createExternalTableFromSql; /** * Saves specified table to the configured bucket. This requires Databricks * cluster to be configured. * * For Azure blob storage you need to configure account access key in * Cluster -> Configuration -> Advanced options * https://docs.databricks.com/data/data-sources/azure/azure-storage.html#access-azure-blob-storage-directly * * `fs.azure.account.key..blob.core.windows.net ` * * For S3 bucket storage you need to configure AWS access key and secret in * Cluster -> Configuration -> Advanced options * https://docs.databricks.com/data/data-sources/aws/amazon-s3.html#access-s3-buckets-directly * * `fs.s3a.access.key ` * `fs.s3a.secret.key ` * * For Google cloud storage you can configure storage credentials and create an external location to access it * or configure account service key (legacy) * https://docs.databricks.com/gcp/en/connect/unity-catalog/cloud-storage/storage-credentials * https://docs.databricks.com/gcp/en/connect/unity-catalog/cloud-storage/external-locations * https://docs.databricks.com/aws/en/connect/storage/gcs */ private createExternalTableFromTable; } export {}; //# sourceMappingURL=DatabricksDriver.d.ts.map