import { AdlsPluginV1 as AdlsPlugin, AthenaPluginV1 as AthenaPlugin, CosmosDbPluginV1 as CosmosDbPlugin, CouchbasePluginV1 as CouchbasePlugin, DatabricksPluginV1 as DatabricksPlugin, KafkaV1 as KafkaPlugin, KinesisPluginV1 as KinesisPlugin, OracleDbPluginV1 as OracleDbPlugin, PluginCommonV1 as PluginCommon, RedisPluginV1 as RedisPlugin, SalesforcePluginV1 as SalesforcePlugin, SmtpPluginV1 as SmtpPlugin } from '@superblocksteam/types'; import { SecretsV1 as Secrets } from '@superblocksteam/types'; import { Property } from '../common/property.js'; import { Plugin, PluginMetadata } from '../plugin/index.js'; import { CreatorDto } from '../rbac/index.js'; import { AWSAuthType, AuthConfig, AuthId, AuthType, IntegrationAuthType, PostgresAuthType, getAuthId } from './auth.js'; export { PluginCommonV1 as PluginCommon } from '@superblocksteam/types'; // The DTO for the full datasource (ie all fields). This should only be visible // to users that can configure the datasource. export class DatasourceDto { id: string; name: string; pluginId: string; pluginName?: string; organizationId?: string; configuration?: DatasourceConfiguration; //TODO(alex): delete configurationProd, configurationStaging once we change an interface between UI and server // @deprecated use configurationProfiles instead configurationProd?: DatasourceConfiguration; // @deprecated use configurationProfiles instead configurationStaging?: DatasourceConfiguration; configurationProfiles?: Record; isDefault?: boolean; creator?: CreatorDto; demoIntegrationId?: string; ownerEmail?: string; error?: string; constructor(datasource: DatasourceDto, ownerEmail?: string, configurationProfiles?: Record) { this.id = datasource.id; this.name = datasource.name; this.pluginId = datasource.pluginId; this.organizationId = datasource.organizationId; this.demoIntegrationId = datasource.demoIntegrationId; this.configuration = datasource.configurationProd; this.configurationProd = datasource.configurationProd; this.configurationStaging = datasource.configurationStaging; this.configurationProfiles = configurationProfiles; this.isDefault = datasource.isDefault; this.ownerEmail = ownerEmail; } } // ViewDatasourceDto is just the information needed to use a datasource to // create steps of that type. Credentials are explicitly excluded - those should // only be needed to execute the step. export class ViewDatasourceDto { id: string; name: string; pluginId: string; organizationId?: string; isDefault?: boolean; readOnly: boolean; ownerEmail?: string; constructor(datasource: DatasourceDto, readOnly: boolean, ownerEmail?: string) { this.id = datasource.id; this.name = datasource.name; this.pluginId = datasource.pluginId; this.organizationId = datasource.organizationId; this.isDefault = datasource.isDefault; this.readOnly = readOnly; this.ownerEmail = ownerEmail; } } export function getAuthIdFromConfig(integrationId: string, config: RestApiIntegrationDatasourceConfiguration): AuthId { if (config.authType) { return getAuthId(config.authType, config.authConfig, integrationId); } return 'empty-auth-type'; } type TemporaryFields = { bindingFields?: string[]; }; export type DynamicWorkflowConfig = { enabled?: boolean; workflowId?: string; }; export type BaseDatasourceConfiguration = PluginMetadata & { name?: string; dynamicWorkflowConfiguration?: DynamicWorkflowConfig; id?: string; }; // Use this aliased typed for default datasources only. For all other // datasources, use BaseDatasourceConfiguration. type DefaultDatasourceConfiguration = BaseDatasourceConfiguration; type LanguageDatasourceConfiguration = DefaultDatasourceConfiguration; interface DBAuthentication { authentication?: { custom?: { databaseName?: Property; }; username?: string; password?: string; }; } type DBDatasourceConfiguration = BaseDatasourceConfiguration & { endpoint?: { host?: string; /** * @type integer * @minimum 0 * @maximum 65536 */ port?: number; }; }; interface DBConnection { connection?: { useSsl?: FakeBoolean; useSelfSignedSsl?: FakeBoolean; ca?: string; key?: string; cert?: string; // This field is set on the demo postgres mode?: 0; }; } interface SSHTunnel { tunnel?: SSHAuthConfiguration; } export enum SharedSSHAuthMethod { PASSWORD = 0, PUB_KEY_RSA = 1, PUB_KEY_ED25519 = 2, USER_PRIVATE_KEY = 3 } type SSHAuthConfiguration = PluginCommon.SSHConfiguration & { authMethod?: SharedSSHAuthMethod }; type AWSDatasourceConfiguration = BaseDatasourceConfiguration & { awsAuthType?: AWSAuthType; authentication?: { [key: string]: unknown; custom?: { region?: Property; accessKeyID?: Property; secretKey?: Property; iamRoleArn?: Property; }; }; }; type BigqueryDatasourceConfiguration = BaseDatasourceConfiguration & { connectionType?: 'fields' | 'service-account-key' | IntegrationAuthType.OAUTH2_TOKEN_EXCHANGE; authTypeField?: string; authConfig?: { tokenUrl?: string; audience?: string; scope?: string; subjectTokenType?: string; subjectTokenSource?: string; subjectTokenSourceStaticToken?: string; workforcePoolId?: string; workforceProviderId?: string; projectId?: string; billingProjectNumber?: string; }; authentication?: { custom?: { googleServiceAccount?: Property; }; }; }; type GCSDatasourceConfiguration = BaseDatasourceConfiguration & { authentication?: { custom?: { googleServiceAccount?: Property; }; }; }; type DynamoDBDatasourceConfiguration = AWSDatasourceConfiguration; type EmailDatasourceConfiguration = BaseDatasourceConfiguration & { authentication?: { custom?: { apiKey?: Property; senderEmail?: Property; senderName?: Property; }; }; // Having the template ID in the datasource config would allow us to // potentially serve different templates for different organizations. sendgridTemplateId?: string; }; type SmtpDatasourceConfiguration = Pick; export type AdlsDatasourceConfiguration = Pick; type SuperblocksOcrDatasourceConfiguration = BaseDatasourceConfiguration & { microsoftComputerVisionApiKey?: string; microsoftComputerVisionResourceBaseUrl?: string; }; //TODO: move all auth methods to this type(service account, API key, DB user/pass etc) export type AuthenticatedDatasourceConfig = BaseDatasourceConfiguration & { authConfig?: AuthConfig; authType?: AuthType; }; export function isAuthenticatedDatasourceConfig(dc: DatasourceConfiguration): dc is AuthenticatedDatasourceConfig { if (!('authConfig' in dc)) { return false; } return 'authType' in dc && dc.authType !== undefined; } type AuthenticatedDatasourceConfigWithDynamicAuthType = BaseDatasourceConfiguration & { authConfig?: AuthConfig; authTypeField: string; [key: string]: string | AuthConfig | undefined; }; export function isAuthenticatedDatasourceConfigWithDynamicAuthType( dc: DatasourceConfiguration ): dc is AuthenticatedDatasourceConfigWithDynamicAuthType { if (!('authConfig' in dc)) { return false; } if ( 'authTypeField' in dc && dc.authTypeField && dc[dc.authTypeField] !== undefined && dc[dc.authTypeField] !== null && dc[dc.authTypeField] !== '' ) { return true; } return false; } /** * Safely extracts the authType from an authenticated datasource configuration. * This function handles both direct authType fields and authTypeField references. * * @param dc - The datasource configuration * @returns The authType if available, undefined otherwise */ export function getAuthTypeFromConfig(dc: DatasourceConfiguration): AuthType | undefined { if (isAuthenticatedDatasourceConfig(dc)) { return dc.authType; } else if (isAuthenticatedDatasourceConfigWithDynamicAuthType(dc)) { return dc[dc.authTypeField] as AuthType; } return undefined; } export type GraphQLDatasourceConfiguration = AuthenticatedHttpDatasourceConfig & { path?: string; }; type JavascriptDatasourceConfiguration = LanguageDatasourceConfiguration; type MariaDBDatasourceConfiguration = DBDatasourceConfiguration & DBConnection & DBAuthentication & { connectionType?: 'fields' | 'url'; connectionUrl?: string; }; type MongoDBDatasourceConfiguration = DBDatasourceConfiguration & DBConnection & DBAuthentication & SSHTunnel; type MySQLDatasourceConfiguration = DBDatasourceConfiguration & DBConnection & DBAuthentication & SSHTunnel & { connectionType?: 'fields' | 'url'; connectionUrl?: string; }; // This type exists for two reasons: // 1. Integration forms sometimes use "checked" instead of a boolean // 2. Our schema generator requires a named type for unions like this type FakeBoolean = boolean | string; // These types are self-contained because ts-json-schema-generator drops value // annotations when the field is composed through a Property intersection. type PostgresDatabaseNameProperty = { key?: string; /** @minLength 1 */ value: string; editable?: boolean; internal?: boolean; system?: boolean; description?: string; mandatory?: boolean; type?: string; defaultValue?: string; minRange?: string; maxRange?: string; valueOptions?: string[]; }; type PostgresIamRoleArnProperty = { key?: string; /** * Validates stable IAM role ARN wire syntax. Role authorization and allowlist * checks remain runtime concerns. The 510-character ARN interior accounts * for the path's leading and trailing slashes within IAM's 512-character max. * @minLength 1 * @pattern ^arn:[a-z0-9-]+:iam::[0-9]{12}:role/(?:[!-~]{1,510}/)?[A-Za-z0-9_+=,.@-]{1,64}$ */ value: string; editable?: boolean; internal?: boolean; system?: boolean; description?: string; mandatory?: boolean; type?: string; defaultValue?: string; minRange?: string; maxRange?: string; valueOptions?: string[]; }; type PostgresIamRegionProperty = { key?: string; /** * Validates region identifier syntax without freezing the set of AWS regions. * Region existence and availability remain runtime concerns. * @minLength 1 * @pattern ^[a-z]{2,}(?:-[a-z0-9]+)+-[0-9]+$ */ value: string; editable?: boolean; internal?: boolean; system?: boolean; description?: string; mandatory?: boolean; type?: string; defaultValue?: string; minRange?: string; maxRange?: string; valueOptions?: string[]; }; type PostgresIamEndpoint = { /** @minLength 1 */ host: string; /** * @type integer * @minimum 1 * @maximum 65535 */ port: number; }; type CockroachDBDatasourceConfiguration = DBDatasourceConfiguration & DBConnection & DBAuthentication & SSHTunnel & { connectionType?: 'fields' | 'url'; connectionUrl?: string; }; type PostgresPasswordDatasourceConfiguration = DBDatasourceConfiguration & DBConnection & SSHTunnel & { authentication?: { authType?: PostgresAuthType.PASSWORD; custom?: { databaseName?: Property; }; password?: string; username?: string; }; connectionType?: 'fields' | 'url'; connectionUrl?: string; }; type PostgresIamRoleDatasourceConfiguration = BaseDatasourceConfiguration & SSHTunnel & { authentication: { authType: PostgresAuthType.AWS_IAM_ROLE; custom: { databaseName: PostgresDatabaseNameProperty; iamRoleArn: PostgresIamRoleArnProperty; region: PostgresIamRegionProperty; sessionPolicy?: Property; }; /** @minLength 1 */ username: string; }; connection: { useSsl: true; useSelfSignedSsl?: FakeBoolean; ca?: string; key?: string; cert?: string; mode?: 0; }; connectionType: 'fields'; endpoint: PostgresIamEndpoint; }; export type PostgresDatasourceConfiguration = PostgresPasswordDatasourceConfiguration | PostgresIamRoleDatasourceConfiguration; type PythonDatasourceConfiguration = LanguageDatasourceConfiguration; type RedshiftDatasourceConfiguration = DBDatasourceConfiguration & DBConnection & { authentication?: { custom?: { databaseName?: Property; databaseSchema?: Property; }; username?: string; password?: string; }; } & { connectionType?: 'fields' | 'url'; connectionUrl?: string; }; export type AuthenticatedHttpDatasourceConfig = AuthenticatedDatasourceConfig & { headers?: Property[]; params?: Property[]; }; export type RestApiDatasourceConfiguration = AuthenticatedHttpDatasourceConfig & { urlBase?: string; // This is part of the in-memory form, not persisted AuthCodeExplanation?: string; FirebaseAlert?: string; FirebaseAlertToken?: string; HTTPBasicAlert?: string; 'oauth-callback-alert'?: string; 'oauth-connect-button'?: string; 'oauth-revoke-shared-tokens-button'?: string; OAuth2PasswordAlert?: string; openApiSpecRef?: string; openApiTenantName?: string; }; type OpenAiDatasourceConfiguration = BaseDatasourceConfiguration & { bearerToken: string; organizationId?: string; }; export type RestApiIntegrationDatasourceConfiguration = RestApiDatasourceConfiguration; type S3DatasourceConfiguration = AWSDatasourceConfiguration; type SnowflakeDatasourceConfiguration = AuthenticatedDatasourceConfig & DBDatasourceConfiguration & DBConnection & { // okta: https://docs.snowflake.com/en/developer-guide/node-js/nodejs-driver-authenticate#use-native-sso-through-okta // key-pair: https://docs.snowflake.com/en/developer-guide/node-js/nodejs-driver-authenticate#label-nodejs-key-pair-authentication // oauth token exchange: https://docs.snowflake.com/en/developer-guide/node-js/nodejs-driver-authenticate#label-nodejs-oauth connectionType?: 'fields' | 'okta' | 'key-pair' | IntegrationAuthType.OAUTH2_TOKEN_EXCHANGE; // if set, will be "connectionType" authTypeField?: string; okta?: { authenticatorUrl?: string; }; keyPair?: { privateKey: string; password?: string; }; authentication?: { username?: string; password?: string; custom?: { databaseName?: Property; account?: Property; warehouse?: Property; schema?: Property; role?: Property; }; }; }; type MsSqlDatasourceConfiguration = DBDatasourceConfiguration & DBConnection & DBAuthentication & { connectionType?: 'fields' | 'url'; connectionUrl?: string; }; type WorkflowDatasourceConfiguration = BaseDatasourceConfiguration; type GoogleSheetsDatasourceConfiguration = RestApiDatasourceConfiguration; // NOTE(frank): Trying to keep these all Protobuf types. // The reason I can't do Pick is because // it wants the class methods to be defined on the type that is defined in this file. // This shoulbe easily fixiable by migrating every instance of PluginMetadata to the Protobuf type. type KafkaDatasourceConfiguration = { superblocksMetadata?: Partial>; } & Pick; type KinesisDatasourceConfiguration = Pick; type AthenaDatasourceConfiguration = Pick; export type RedisDatasourceConfiguration = Pick; // NOTE(frank): We should just be using `SecretV1.Store`. However, ts-json-schema-generator does not // support `bigint`. So you'd think I can just do `Omit`. However, it doesn't // respect Pick, Omit, or Exclude.... Hence, this workaround where we literally dupe the type.... type SecretStore = { provider?: Secrets.Provider; ttl?: number; configurationId: string; }; export type CosmosDbDatasourceConfiguration = Pick; export type CouchbaseDatasourceConfiguration = Pick< CouchbasePlugin.Plugin, 'name' | 'connection' | 'tunnel' | 'dynamicWorkflowConfiguration' >; type DatabricksDatasourceConfiguration = Pick & AuthenticatedDatasourceConfig & { authTypeField?: string; oauthType?: IntegrationAuthType.OAUTH2_TOKEN_EXCHANGE; }; // Lakebase is Databricks's Postgres-compatible database export type LakebaseDatasourceConfiguration = { name?: string; connection?: { host?: string; port?: number; databaseName?: string; connectionType?: string; username?: string; password?: string; oauthClientId?: string; oauthClientSecret?: string; oauthWorkspaceUrl?: string; }; } & AuthenticatedDatasourceConfig & { authTypeField?: string; oauthType?: IntegrationAuthType.OAUTH2_TOKEN_EXCHANGE; }; // Snowflake Postgres is Snowflake's Postgres-compatible database endpoint export type SnowflakePostgresDatasourceConfiguration = { name?: string; connection?: { host?: string; port?: number; databaseName?: string; username?: string; password?: string; useSelfSignedSsl?: boolean; ca?: string; key?: string; cert?: string; }; connectionType?: 'fields' | 'url'; connectionUrl?: string; }; export type OracleDbDatasourceConfiguration = Pick & { connectionType?: 'fields' | 'url'; connectionUrl?: string; }; // authType and authToken is also defined in salesforce template // this will be populated by orchestrator, and it will be used by plugin.execute | metadata | test // so the plugin will get ready-to-use oauth token. This is needed until we figure out a way to pass token // to plugin.metadata and plugin.test export type SalesforceDatasourceConfiguration = AuthenticatedDatasourceConfig & { // if set, will be set to 'connection.auth.method.case' authTypeField?: string; authTypeOnBehalfOfToken?: string; } & Pick; export type DatasourceConfiguration = ( | AthenaDatasourceConfiguration | BigqueryDatasourceConfiguration | CockroachDBDatasourceConfiguration | CosmosDbDatasourceConfiguration | DynamoDBDatasourceConfiguration | EmailDatasourceConfiguration | GCSDatasourceConfiguration | GraphQLDatasourceConfiguration | JavascriptDatasourceConfiguration | MariaDBDatasourceConfiguration | MongoDBDatasourceConfiguration | MySQLDatasourceConfiguration | OpenAiDatasourceConfiguration | PostgresDatasourceConfiguration | PythonDatasourceConfiguration | RedisDatasourceConfiguration | RedshiftDatasourceConfiguration | RestApiDatasourceConfiguration | RestApiIntegrationDatasourceConfiguration | S3DatasourceConfiguration | SnowflakeDatasourceConfiguration | SuperblocksOcrDatasourceConfiguration | WorkflowDatasourceConfiguration | MsSqlDatasourceConfiguration | GoogleSheetsDatasourceConfiguration | KafkaDatasourceConfiguration | SalesforceDatasourceConfiguration | SmtpDatasourceConfiguration | AdlsDatasourceConfiguration | OracleDbDatasourceConfiguration | DatabricksDatasourceConfiguration | LakebaseDatasourceConfiguration | SnowflakePostgresDatasourceConfiguration | CouchbaseDatasourceConfiguration | SecretStore ) & TemporaryFields; export type DatasourceConfigurationByType = { adls?: AdlsDatasourceConfiguration & TemporaryFields; aivenkafka?: KafkaDatasourceConfiguration & TemporaryFields; athena?: AthenaDatasourceConfiguration & TemporaryFields; bigquery?: BigqueryDatasourceConfiguration & TemporaryFields; cockroachdb?: CockroachDBDatasourceConfiguration & TemporaryFields; confluent?: KafkaDatasourceConfiguration & TemporaryFields; couchbase?: CouchbaseDatasourceConfiguration & TemporaryFields; databricks?: DatabricksDatasourceConfiguration & TemporaryFields; dynamodb?: DynamoDBDatasourceConfiguration & TemporaryFields; email?: EmailDatasourceConfiguration & TemporaryFields; gcs?: GCSDatasourceConfiguration & TemporaryFields; graphqlintegration?: GraphQLDatasourceConfiguration & TemporaryFields; graphql?: GraphQLDatasourceConfiguration & TemporaryFields; javascript?: JavascriptDatasourceConfiguration & TemporaryFields; lakebase?: LakebaseDatasourceConfiguration & TemporaryFields; mariadb?: MariaDBDatasourceConfiguration & TemporaryFields; mongodb?: MongoDBDatasourceConfiguration & TemporaryFields; msk?: KafkaDatasourceConfiguration & TemporaryFields; mysql?: MySQLDatasourceConfiguration & TemporaryFields; openai?: OpenAiDatasourceConfiguration & TemporaryFields; oracledb?: OracleDbDatasourceConfiguration & TemporaryFields; postgres?: PostgresDatasourceConfiguration & TemporaryFields; python?: PythonDatasourceConfiguration & TemporaryFields; redpanda?: KafkaDatasourceConfiguration & TemporaryFields; redshift?: RedshiftDatasourceConfiguration & TemporaryFields; restapi?: RestApiDatasourceConfiguration & TemporaryFields; restapiintegration?: RestApiIntegrationDatasourceConfiguration & TemporaryFields; s3?: S3DatasourceConfiguration & TemporaryFields; smtp?: SmtpDatasourceConfiguration & TemporaryFields; snowflake?: SnowflakeDatasourceConfiguration & TemporaryFields; snowflakecortex?: RestApiIntegrationDatasourceConfiguration & TemporaryFields; snowflakepostgres?: SnowflakePostgresDatasourceConfiguration & TemporaryFields; salesforce?: SalesforceDatasourceConfiguration & TemporaryFields; 'superblocks-ocr'?: SuperblocksOcrDatasourceConfiguration & TemporaryFields; workflow?: WorkflowDatasourceConfiguration & TemporaryFields; mssql?: MsSqlDatasourceConfiguration & TemporaryFields; gsheets?: GoogleSheetsDatasourceConfiguration & TemporaryFields; kafka?: KafkaDatasourceConfiguration & TemporaryFields; kinesis?: KinesisDatasourceConfiguration & TemporaryFields; }; export enum DatasourceEnvironments { PRODUCTION = 'Production', STAGING = 'Staging', // Only for V1 orgs, V2 orgs use DEV DEV = 'Dev' } export const ENVIRONMENT_PRODUCTION = 'production'; export const ENVIRONMENT_STAGING = 'staging'; // Only for V1 orgs, V2 orgs use ENVIRONMENT_DEV export const ENVIRONMENT_DEV = 'dev'; export const ENVIRONMENT_ALL = '*'; export const VALID_DATASOURCE_ENVIRONMENTS = [ENVIRONMENT_PRODUCTION, ENVIRONMENT_DEV, ENVIRONMENT_STAGING, ENVIRONMENT_ALL]; export type Integration = { datasource: DatasourceDto; plugin: Plugin; }; export * from './auth.js'; export * from './metadata/index.js'; export * from './test.js';