/** * @fileoverview Generic Database Provider for MemberJunction * * This module provides an intermediate abstract base class between DatabaseProviderBase (MJCore) * and platform-specific providers (SQLServerDataProvider, PostgreSQLDataProvider). * * It contains shared logic that requires heavy dependencies (ActionEngine, AIEngine, * EncryptionEngine, MJCoreEntities) which cannot live in MJCore's lightweight base class. * * Inheritance chain: * DatabaseProviderBase (MJCore — no heavy deps) * └── GenericDatabaseProvider (this package — has ActionEngine, AIEngine, EncryptionEngine) * ├── SQLServerDataProvider (adds datetime handling, SQL logging, mssql-specific) * └── PostgreSQLDataProvider (adds pg-specific connection management) * * @module @memberjunction/generic-database-provider */ import { BaseEntity, DatabaseProviderBase, EntityInfo, EntityFieldInfo, EntitySaveOptions, EntityDeleteOptions, EntityPermissionType, ILocalStorageProvider, RunViewParams, RunViewResult, RunViewWithCacheCheckParams, RunViewsWithCacheCheckResponse, RunViewWithCacheCheckResult, RunQueryParams, RunQueryResult, RunQueryWithCacheCheckParams, RunQueriesWithCacheCheckResponse, RunQueryWithCacheCheckResult, QueryCacheAuthorization, CompositeKey, DatasetItemFilterType, DatasetResultType, DatasetStatusResultType, IMetadataProvider, UserInfo, QueryExecutionSpec, SaveContext, SaveSQLResult, ExternalDataSourceReadRouter } from '@memberjunction/core'; import { SqlLoggingOptions, SqlLoggingSession } from './types.js'; import { SQLDialect } from '@memberjunction/sql-dialect'; import { type RenderResult } from './renderPipeline.js'; import { CRUDSprocType } from './crudSprocFieldRules.js'; import { SaveCoercedValue, SaveCallBinding, SaveSQLFragment } from './saveTypes.js'; import type { RecordChangePayload } from '@memberjunction/core'; import { MJEntityAIActionEntity, MJQueryEntityExtended, MJUserViewEntityExtended } from '@memberjunction/core-entities'; import { EntityAIActionParams } from '@memberjunction/aiengine'; import { ScoredCandidate } from '@memberjunction/core'; import { ActionResult } from '@memberjunction/actions-base'; /** * Configuration options for batch SQL execution. * Shared between GenericDatabaseProvider and platform-specific providers. */ export interface ExecuteSQLBatchOptions { /** Optional description for this batch operation */ description?: string; /** If true, this batch will not be logged to any logging session */ ignoreLogging?: boolean; /** Whether this batch contains data mutation operations */ isMutation?: boolean; } export declare abstract class GenericDatabaseProvider extends DatabaseProviderBase { /**************************************************************************/ /**************************************************************************/ /** * The local storage provider backing server-side caching for metadata, * RunView results, RunQuery results, and datasets. * * Defaults to {@link InMemoryLocalStorageProvider} (data lost on restart, not shared * across instances). To enable persistent, shared caching, replace with a * {@link https://www.npmjs.com/package/@memberjunction/redis-provider | RedisLocalStorageProvider} * or any other {@link ILocalStorageProvider} implementation. * * Subclasses can override this property to supply a custom provider, or call * {@link SetLocalStorageProvider} to swap the provider at runtime (e.g., after * reading configuration). * * @see {@link ILocalStorageProvider} for the interface contract * @see {@link InMemoryLocalStorageProvider} for the default in-memory implementation */ private _localStorageProvider; /** * Returns the active local storage provider, lazily creating an * {@link InMemoryLocalStorageProvider} if none has been set. * * This fulfills the abstract `LocalStorageProvider` requirement from * {@link ProviderBase} and is shared by all database providers * (SQL Server, PostgreSQL, and any future platforms). */ get LocalStorageProvider(): ILocalStorageProvider; /** * Replaces the active local storage provider at runtime. * * Use this to swap from the default in-memory provider to a Redis-backed * provider (or any other {@link ILocalStorageProvider}) after reading * application configuration. * * @param provider - The new storage provider to use for all caching operations. * * @example * ```typescript * import { RedisLocalStorageProvider } from '@memberjunction/redis-provider'; * * // During server startup, after config is loaded: * const redis = new RedisLocalStorageProvider({ * url: process.env.REDIS_URL, * defaultTTLSeconds: 300 * }); * (Metadata.Provider as GenericDatabaseProvider).SetLocalStorageProvider(redis); * ``` */ SetLocalStorageProvider(provider: ILocalStorageProvider): void; /**************************************************************************/ /**************************************************************************/ private static _sqlLoggingSessionsKey; private get _sqlLoggingSessions(); /** * Creates a new SQL logging session that will capture all SQL operations to a file. * Returns a disposable session object that must be disposed to stop logging. * * @param filePath - Full path to the file where SQL statements will be logged * @param options - Optional configuration for the logging session * @returns Promise - Disposable session object * * @example * ```typescript * // Basic usage * const session = await provider.CreateSqlLogger('./logs/metadata-sync.sql'); * try { * // Perform operations that will be logged * await provider.ExecuteSQL('INSERT INTO ...'); * } finally { * await session.dispose(); // Stop logging * } * * // With migration formatting * const session = await provider.CreateSqlLogger('./migrations/changes.sql', { * formatAsMigration: true, * description: 'MetadataSync push operation' * }); * ``` */ CreateSqlLogger(filePath: string, options?: SqlLoggingOptions): Promise; /** * Gets information about all active SQL logging sessions. * Useful for monitoring and debugging. * * @returns Array of session information objects */ GetActiveSqlLoggingSessions(): Array<{ id: string; filePath: string; startTime: Date; statementCount: number; options: SqlLoggingOptions; }>; /** * Gets a specific SQL logging session by its ID. * Returns the session if found, or undefined if not found. * * @param sessionId - The unique identifier of the session to retrieve * @returns The SqlLoggingSession if found, undefined otherwise */ GetSqlLoggingSessionById(sessionId: string): SqlLoggingSession | undefined; /** * Disposes all active SQL logging sessions. * Useful for cleanup on provider shutdown. */ DisposeAllSqlLoggingSessions(): Promise; /** * Internal method to log SQL statement to all active logging sessions. * This is called automatically by ExecuteSQL methods. * Protected so platform-specific providers can reference it (e.g., to bind as a callback). * * @param query - The SQL query being executed * @param parameters - Parameters for the query * @param description - Optional description for this operation * @param ignoreLogging - If true, this statement will not be logged * @param isMutation - Whether this is a data mutation operation * @param simpleSQLFallback - Optional simple SQL to use for loggers with logRecordChangeMetadata=false * @param contextUser - Optional user context for session filtering */ protected _logSqlStatement(query: string, parameters?: unknown, description?: string, ignoreLogging?: boolean, isMutation?: boolean, simpleSQLFallback?: string, contextUser?: UserInfo): Promise; /** * Static method to log SQL statements from external sources like transaction groups. * Gets the current provider instance from Metadata.Provider and delegates to the * instance _logSqlStatement method. * * @param query - The SQL query being executed * @param parameters - Parameters for the query * @param description - Optional description for this operation * @param isMutation - Whether this is a data mutation operation * @param simpleSQLFallback - Optional simple SQL to use for loggers with logRecordChangeMetadata=false * @param contextUser - Optional user context for session filtering */ static LogSQLStatement(query: string, parameters?: unknown, description?: string, isMutation?: boolean, simpleSQLFallback?: string, contextUser?: UserInfo): Promise; /**************************************************************************/ /**************************************************************************/ /** * Returns AI actions configured for the given entity and timing. * Uses AIEngine metadata to find matching EntityAIAction records. */ protected GetEntityAIActions(entityInfo: EntityInfo, before: boolean): MJEntityAIActionEntity[]; /** * Handles entity actions (non-AI) for save, delete, or validate operations. * Uses EntityActionEngineServer to discover and run active actions. */ protected HandleEntityActions(entity: BaseEntity, baseType: 'save' | 'delete' | 'validate', before: boolean, user: UserInfo): Promise; /** * Handles Entity AI Actions for save or delete operations. * * For "before save" actions: blocks (awaits) until complete. * For "after save" actions: fires and forgets via QueueManager. * * Subclasses that manage transactions can override to defer after-save tasks * until after transaction commit (see SQLServerDataProvider). */ protected HandleEntityAIActions(entity: BaseEntity, baseType: 'save' | 'delete', before: boolean, user: UserInfo): Promise; /** * Enqueues an after-save AI action for execution. By default, immediately adds * to QueueManager. Subclasses with transaction support can override to defer * until after transaction commit. */ protected EnqueueAfterSaveAIAction(params: EntityAIActionParams, user: UserInfo): void; /**************************************************************************/ /**************************************************************************/ protected OnValidateBeforeSave(entity: BaseEntity, user: UserInfo): Promise; protected OnBeforeSaveExecute(entity: BaseEntity, user: UserInfo, options: EntitySaveOptions, context: SaveContext): Promise; protected OnAfterSaveExecute(entity: BaseEntity, user: UserInfo, options: EntitySaveOptions, _context: SaveContext): void; protected OnSaveCompleted(entity: BaseEntity, saveSQLResult: SaveSQLResult, user: UserInfo, options: EntitySaveOptions, context: SaveContext): Promise | null>; protected OnBeforeDeleteExecute(entity: BaseEntity, user: UserInfo, options: EntityDeleteOptions): Promise; protected OnAfterDeleteExecute(entity: BaseEntity, user: UserInfo, options: EntityDeleteOptions): void; /** * Delete RecordGeoCode rows associated with a deleted entity record. */ private cleanupGeoCodesForDeletedRecord; /**************************************************************************/ /**************************************************************************/ /** * Post-processes rows: first applies platform-specific datetime adjustments * via the virtual `AdjustDatetimeFields` hook, then handles field-level * decryption for encrypted fields. * * Subclasses should NOT override this method. Instead, override * `AdjustDatetimeFields` for platform-specific datetime corrections. */ protected PostProcessRows(rows: Record[], entityInfo: EntityInfo, user: UserInfo): Promise[]>; /**************************************************************************/ /**************************************************************************/ /** * Encrypts field values before saving to the database. * * This method handles field-level encryption for any entity with encrypted fields. * It is called by platform-specific providers (SQL Server, PostgreSQL) before * building their SQL parameters, ensuring encryption is handled generically. * * For each field marked with `Encrypt=true`: * - If `EncryptionKeyID` is null → throws an error (misconfiguration) * - If value is null/undefined → skips (nothing to encrypt) * - If value is already encrypted → skips (prevents double-encryption) * - Otherwise → encrypts the value using the configured key * * @param entity The entity being saved * @param fieldValues Map of field info to current values — values are mutated in-place * @param contextUser User context for encryption operations * @throws Error if a field is marked for encryption but has no EncryptionKeyID configured * @throws Error if encryption fails (to prevent storing unencrypted sensitive data) */ protected EncryptFieldValuesForSave(entity: BaseEntity, fieldValues: Map, contextUser?: UserInfo): Promise; /** * Virtual hook for platform-specific datetime field adjustments. * Default implementation is a no-op (returns rows unchanged). * * SQL Server overrides this to correct datetime2/datetimeoffset/datetime * timezone interpretation issues in the mssql driver. * PostgreSQL does NOT need to override — PG timestamp types are timezone-aware natively. * * @param rows The data rows to process * @param datetimeFields Entity fields with TSType === Date * @param entityInfo The entity metadata * @returns The rows with datetime fields adjusted (or unchanged for default) */ protected AdjustDatetimeFields(rows: Record[], datetimeFields: EntityFieldInfo[], entityInfo: EntityInfo): Promise[]>; /**************************************************************************/ /**************************************************************************/ /** * Executes multiple SQL queries and returns an array of result arrays, one per query. * * The default implementation runs queries in parallel using `Promise.all(ExecuteSQL(...))`. * Platform-specific providers can override for true multi-result-set batching: * - SQL Server: concatenates queries and uses a single mssql request with multiple recordsets * - PostgreSQL: could use pg pipeline or simple parallel execution * * @param queries Array of SQL query strings to execute * @param parameters Optional array of parameter arrays, one per query * @param options Optional batch execution options * @param contextUser Optional user context for logging/filtering * @returns Array of result arrays, one for each query */ ExecuteSQLBatch(queries: string[], parameters?: unknown[][], options?: ExecuteSQLBatchOptions, contextUser?: UserInfo): Promise[][]>; /**************************************************************************/ /**************************************************************************/ /**************************************************************************/ /**************************************************************************/ /** * Returns the SQLDialect instance for this provider's platform. * Subclasses override to return the appropriate dialect (e.g. SQLServerDialect, PostgreSQLDialect). * Used by `PlatformBatchSeparator` to retrieve the correct batch separator token via * `@memberjunction/sql-dialect` rather than hardcoding platform strings. */ protected getDialect(): SQLDialect | null; /** * Detects infrastructure-level connection errors (timeout, refused, pool closed) * as opposed to query-level errors (bad SQL, constraint violations). * Delegates to the dialect's driver-specific error classification, with a * fallback for POOL_CLOSED errors thrown by our own code. */ protected isConnectionError(e: unknown): boolean; /** * Returns the batch separator token for the underlying database platform by delegating to * the SQLDialect instance returned by `getDialect()`. * SQL Server → `'GO'`, PostgreSQL → `''` (no separator needed). * Auto-injected as the default `batchSeparator` in `CreateSqlLogger`. */ protected get PlatformBatchSeparator(): string; /** * Maximum number of parameters a CRUD stored procedure (`spCreate*`/`spUpdate*`/`spDelete*`) * may declare on this database platform before CodeGen must emit a JSON-arg shape instead. * * - SQL Server: `Infinity` — SS supports up to 2,100 parameters per procedure, far beyond * any realistic CRUD sproc. * - PostgreSQL: `90` — PG's hard `FUNC_MAX_ARGS` ceiling is 100 (compiled into the server, * not configurable on managed services like RDS/Aurora). 90 leaves headroom for column * adds without flipping sproc shape unexpectedly. * * Used by the shared `useJsonArgShape` predicate (CodeGen + provider call-site) to decide * whether a given sproc should emit as typed-args + `_Clear` companions (today's shape) or * as a single-JSONB-arg sproc with key-presence tri-state semantics. See * [plans/json-arg-crud-sprocs.md](../../../plans/json-arg-crud-sprocs.md) and GitHub * issue #2552. */ get ProcedureParamLimit(): number; /** * Returns true when CRUD sproc generation and call-construction for the * given entity + sproc verb should use a single JSON-arg shape (instead of * typed args + `_Clear` companions). * * Convenience wrapper around the pure `useJsonArgShape` helper, applying * this provider's `ProcedureParamLimit`. CodeGen and runtime call-sites * can invoke this via the provider instance to keep sproc emit and sproc * invocation in lockstep. */ UseJsonArgShape(entity: EntityInfo, sprocType: CRUDSprocType): boolean; /**************************************************************************/ /**************************************************************************/ /** * Per-field value transform applied before encryption + rendering. Each * provider owns its type-coercion rules: * * - SQL Server: `datetimeoffset → ISO string`; non-PK `uniqueidentifier` * with `()` function-literal → `skip` (let DB default fire). * - PostgreSQL: `gen_random_uuid()`/`uuid_generate_v4()` → fresh UUID; * `NOW()`/`CURRENT_TIMESTAMP` → `null` (let DB default fire); bool / * binary / Date pass-throughs handled here. * * Returns `{ kind: 'use', value }` to bind the (possibly transformed) * value, or `{ kind: 'skip' }` to omit the field entirely. * * Runs in `GenerateSaveSQL` *before* `EncryptFieldValuesForSave` and * *before* `RenderSaveCallBinding`. */ protected abstract CoerceSaveFieldValue(field: EntityFieldInfo, value: unknown, isUpdate: boolean): SaveCoercedValue; /** * Renders the dialect-specific parameter binding for a save call. * Returns a discriminated `SaveCallBinding` the orchestrator treats as * opaque — the same provider's `WrapSaveCallForResult` / * `WrapSaveCallWithRecordChange` pattern-match the variant. * * `fieldValues` is the post-coercion, post-encryption map from * `GenerateSaveSQL`. Iteration order matters for PostgreSQL positional * binding — Map iteration follows insertion order, which the * orchestrator preserves. */ protected abstract RenderSaveCallBinding(entity: BaseEntity, fieldValues: Map, isUpdate: boolean, spName: string): SaveCallBinding; /** * Wraps the bare SP-call binding with the dialect's result-capture * pattern. SQL Server emits `DECLARE @ResultTable + INSERT INTO @ResultTable EXEC + SELECT * FROM @ResultTable`; * PostgreSQL emits the bare `SELECT * FROM schema."fn"(...)`. */ protected abstract WrapSaveCallForResult(binding: SaveCallBinding, entity: BaseEntity, spName: string): SaveSQLFragment; /** * Wraps an already-result-captured save SQL with the dialect's * record-change emission. SQL Server inlines `EXEC spCreateRecordChange_Internal` * after the result table; PostgreSQL emits a CTE chain with * `INSERT INTO "RecordChange" ... RETURNING`. * * Orchestrator only calls this when `ShouldTrackRecordChanges` is true * and `BuildRecordChangePayload` returned a non-null payload. */ protected abstract WrapSaveCallWithRecordChange(saveSQL: SaveSQLFragment, binding: SaveCallBinding, payload: RecordChangePayload, entity: BaseEntity): SaveSQLFragment; /** * Concrete implementation of the abstract save-SQL builder defined on * `DatabaseProviderBase`. Iterates fields via the single `IsSPParameter` * predicate, applies provider-specific value coercion, encrypts, then * delegates parameter binding, result-capture wrapping, and record-change * wrapping to abstract hooks the provider subclass implements. * * See `plans/sp-save-builder-generic-layer-refactor.md` (rev 4) for * the design and the rev-3 lesson that motivated this shape. */ protected GenerateSaveSQL(entity: BaseEntity, isNew: boolean, user: UserInfo): Promise; /** * Builds a platform-specific pagination clause. * SQL Server: `OFFSET X ROWS FETCH NEXT Y ROWS ONLY` * PostgreSQL: `LIMIT Y OFFSET X` */ protected abstract BuildPaginationSQL(maxRows: number, startRow: number): string; /** * Builds a platform-specific TOP/LIMIT clause for non-paginated row limits. * SQL Server: `TOP N`; PostgreSQL returns empty (uses LIMIT via BuildPaginationSQL). * Default: returns empty string. SQL Server overrides. */ protected BuildTopClause(_maxRows: number): string; /** * Builds a platform-specific non-paginated row limit clause appended at end of query. * SQL Server: returns '' (already handled by TOP in SELECT clause). * PostgreSQL: returns `LIMIT N`. * Default: returns empty string. PG overrides. */ protected BuildNonPaginatedLimitSQL(_maxRows: number): string; /** * Builds the `SELECT COUNT(*) AS TotalRowCount FROM ...` SQL used to compute the total * row count for paginated views. Returns `null` when the view isn't row-limited (no count * query needed). * * Two PG-parity details this method encapsulates — both caught real regressions: * * 1. **When to emit the count query.** Returns non-null whenever rows are being limited * — either explicit pagination or MaxRows/UserViewMaxRows. Earlier code keyed off * `topSQL.length > 0`, which was SQL-Server-specific: PG's `BuildTopClause` returns * empty (PG uses `LIMIT` appended at end via `BuildNonPaginatedLimitSQL`, not `TOP` * in the SELECT). So the old condition missed every PG case where MaxRows was set * without explicit StartRow — Explorer's Entity list was one such case and showed * "100 of 100" (no pagination) instead of "100 of 299". * * 2. **Quote the alias via `QuoteIdentifier`.** PostgreSQL folds unquoted column aliases * to lowercase, so `AS TotalRowCount` returns a row keyed `totalrowcount` on PG. The * caller reads `countResult[0].TotalRowCount` (PascalCase) and gets `undefined` — the * count falls back to `retData.length` (the page size), so pagination breaks silently. * SQL Server is case-insensitive so it worked unquoted there. Quoting via * `QuoteIdentifier` produces `"TotalRowCount"` on PG and `[TotalRowCount]` on SQL * Server — both preserve case. */ protected BuildTotalRowCountSQL(entityInfo: EntityInfo, usingPagination: boolean, maxRowsForQuery: number): string | null; /** * Validates that the entity and RunViewParams are compatible with keyset (AfterKey) pagination, * then returns the SQL predicate (` > 'value'` or ` < 'value'`) and the resolved * order-by direction. * * See {@link AfterKeyNotSupportedError} for the validation rules and {@link RunViewParams.AfterKey} * for the API contract. * * @throws AfterKeyNotSupportedError on validation failure */ protected BuildKeysetSeekClause(entityInfo: EntityInfo, params: RunViewParams): { seekPredicate: string; direction: 'ASC' | 'DESC'; pkColumnName: string; }; /** * Formats a CompositeKey value as a SQL literal for use in a keyset seek predicate. * * This bypasses parameter binding because the entire WHERE clause is built as a string * elsewhere in this provider (consistent with ExtraFilter / OrderBy handling). The seek * value comes from server-side application code, not raw user input — but we still type- * check and escape to defend against any caller passing a tainted value. * * Strategy: * - UUID types: validate strict UUID format, wrap in single quotes * - Numeric types: validate finite number, output bare * - Boolean: output 0/1 (SQL Server) or TRUE/FALSE (caller can override if needed) * - Date/time types: validate ISO-ish string, wrap in single quotes * - String types: escape single quotes by doubling, wrap in single quotes * * @throws AfterKeyNotSupportedError if the value fails type-specific validation */ protected formatKeysetSeekValue(value: unknown, sqlType: string): string; /** * Transforms a user-provided SQL clause (ExtraFilter, OrderBy, etc.) for platform compatibility. * PostgreSQL overrides to quote mixed-case identifiers and convert bracket notation. * Default: returns the clause unchanged. */ protected TransformExternalSQLClause(clause: string, _entityInfo: EntityInfo): string; /** * Optionally wraps a view query with user view run logging. * SQL Server overrides to use spCreateUserViewRunWithDetail. * Default: returns null (no view run logging). */ protected executeSQLForUserViewRunLogging(_viewId: number, _entityBaseView: string, _whereSQL: string, _orderBySQL: string, _user: UserInfo): Promise<{ executeViewSQL: string; runID: string; } | null>; /** * Shared InternalRunView implementation. * Handles: view resolution, permissions, field selection, WHERE clause building * (view + extra filter + user search + exclude + RLS), ORDER BY, pagination, * aggregates, parallel query execution, post-processing, and audit logging. */ protected InternalRunView(params: RunViewParams, contextUser?: UserInfo): Promise>; protected InternalRunViews(params: RunViewParams[], contextUser?: UserInfo): Promise[]>; /**************************************************************************/ /**************************************************************************/ /** * Builds the SQL field list string for a view query, using dialect-neutral quoting. * Returns '*' if no specific fields are resolved. */ protected getRunTimeViewFieldString(params: RunViewParams, viewEntity: MJUserViewEntityExtended | null): string; /** * Resolves the list of EntityFieldInfo objects for a view query. * Priority: params.Fields > view columns > all entity fields (wildcard). */ protected getRunTimeViewFieldArray(params: RunViewParams, viewEntity: MJUserViewEntityExtended | null): EntityFieldInfo[]; /** * Builds user search SQL for the given entity and search string. * * Supports full-text search (if enabled) and field-by-field LIKE searching. * For the LIKE path: * - honors EntityField.UserSearchPredicateAPI (Exact / BeginsWith / EndsWith / Contains) * so index-seekable predicates can be expressed, * - escapes LIKE metacharacters (%, _, [, ]) in user input with ESCAPE '\\', * - skips fields that are not sensible text-search targets (non-text types, * unbounded text columns when FTX is off). */ protected createViewUserSearchSQL(entityInfo: EntityInfo, userSearchString: string): string; /** * Build the SQL fragment that compares one EntityField against a user search term. * * Resolution order: * 1. UserSearchParamFormatAPI (custom override) wins if set, with `{0}` replaced * by the single-quote-escaped raw term. Caller is responsible for escaping * LIKE metacharacters in their format string if needed. * 2. Otherwise, the field must be a text-searchable type. Non-text and unbounded- * text fields return '' (caller skips them). * 3. Otherwise, the predicate is chosen from UserSearchPredicateAPI: * Exact -> = N'term' (index-seekable) * BeginsWith -> LIKE N'term%' ESCAPE '\\' (index-seekable) * EndsWith -> LIKE N'%term' ESCAPE '\\' * Contains -> LIKE N'%term%' ESCAPE '\\' (default, non-seekable) */ protected buildPerFieldSearchPredicate(field: EntityFieldInfo, escapedTerm: string, rawSafeTerm: string): string; /** * Escape characters that have special meaning in SQL Server LIKE patterns. * The backslash itself must be escaped first so its replacement isn't reprocessed. * Pair with `ESCAPE '\\'` on the LIKE clause. */ protected escapeLikeTerm(safeTerm: string): string; /** * True if the field's column type is appropriate for text-pattern search. * Non-text types are rejected because LIKE forces an implicit per-row CONVERT * to nvarchar. Unbounded (MAX/ntext/text) columns are rejected because the * LIKE path cannot seek them; FTX is the right tool for those. */ protected isTextSearchableType(field: EntityFieldInfo): boolean; /**************************************************************************/ /**************************************************************************/ /** * Renders the WHERE clause for a saved view, replacing template variables * like {%UserView "viewId"%} with subquery SQL. Handles nested/recursive * templates with circular reference detection. * * Uses QuoteIdentifier/QuoteSchemaAndView for dialect-neutral SQL generation. */ protected RenderViewWhereClause(viewEntity: MJUserViewEntityExtended, user: UserInfo, stack?: string[]): Promise; /**************************************************************************/ /**************************************************************************/ /** * Parses a timestamp string sent by the client. * * Accepts both ISO 8601 strings (the canonical form) and all-digit strings * representing milliseconds since epoch. The numeric form has been observed * in the wild from clients whose cache layer round-tripped a Date through a * lossy serializer — `new Date('1778004618383')` returns `Invalid Date`, * but `new Date(Number('1778004618383'))` is a valid timestamp. Returning * `null` on unparseable input lets callers degrade to a stale-cache fallback * instead of throwing `RangeError: Invalid time value` on a downstream * `.toISOString()`. */ protected parseClientTimestamp(raw: string | null | undefined): Date | null; /** * Compares client cache status with server status to determine if cache is current. * Checks both row count and maxUpdatedAt timestamp. */ protected isCacheCurrent(clientStatus: { maxUpdatedAt: string; rowCount: number; }, serverStatus: { maxUpdatedAt?: string; rowCount?: number; }): boolean; /**************************************************************************/ /**************************************************************************/ /** * Smart cache validation for batch RunViews. * For each view request, if cacheStatus is provided, checks if the cache is current * by comparing MAX(__mj_UpdatedAt) and COUNT(*) with client's values. * Returns 'current' if cache is valid (no data), 'stale' with fresh data, or 'differential' * with only changed rows for entities that track record changes. */ RunViewsWithCacheCheck(params: RunViewWithCacheCheckParams[], contextUser?: UserInfo): Promise>; /** * Builds the WHERE clause for cache status check, using same logic as InternalRunView. * Handles ExtraFilter, UserSearch, and Row-Level Security. * Subclasses can override to add platform-specific SQL transformations (e.g., identifier quoting). */ protected buildWhereClauseForCacheCheck(params: RunViewParams, entityInfo: EntityInfo, user: UserInfo): Promise; /** * Executes cache status checks for multiple views. * Default: parallel individual queries (works on all platforms). * SQL Server overrides to use ExecuteSQLBatch for multi-result-set efficiency. */ protected getBatchedServerCacheStatus(items: Array<{ index: number; item: RunViewWithCacheCheckParams; entityInfo: EntityInfo; whereSQL: string; }>, contextUser?: UserInfo): Promise>; /** * Runs a full view query and returns results with cache metadata. */ protected runFullQueryAndReturn(params: RunViewParams, viewIndex: number, contextUser?: UserInfo): Promise>; /** * Runs a full query and stores the result in the server's LocalCacheManager. * Used by RunViewsWithCacheCheck to populate the server cache for future requests. */ protected runFullQueryAndCacheResult(params: RunViewParams, viewIndex: number, contextUser?: UserInfo, callerFields?: string[] | null): Promise>; /** * Resolves the cache TTL (in ms) for a RunView's entity, or `undefined` for MJ-DB entities * (which use event-based cache invalidation, not TTL). External-data-source entities MUST be * time-bounded — their remote data changes never emit BaseEntity events — so this returns the * data source's DefaultCacheTTLSeconds (via the external router) in ms, or `0` to signal "do * not cache" (TTL disabled on the source, or no router available to resolve one). */ protected resolveExternalCacheTTLMs(params: RunViewParams, contextUser?: UserInfo): Promise; /** * Resolve the External Data Sources read router, or throw a CLEAR error when the EDS engine isn't * loaded. IMPORTANT: `ClassFactory.CreateInstance` returns an instance of the ABSTRACT base class * (not null) when no concrete class is registered, so a `if (!router)` guard is dead code and the * caller would instead hit a cryptic "router.X is not a function" TypeError. Gate on GetRegistration. */ private resolveExternalReadRouterOrThrow; /** * Guards external-data-source reads against silently bypassing Row-Level Security. A remote * system can't enforce MJ's RLS WHERE clauses, so if RLS would filter this user's rows we * refuse the read with a clear error rather than returning unfiltered data. Users exempt from * RLS (e.g. admins) get an empty clause and pass through — RLS wouldn't restrict them on an * MJ-DB entity either. Called from the external RunView and Load dispatch points. */ protected assertExternalReadAllowedUnderRLS(entityInfo: EntityInfo, user: UserInfo): void; /** * Rejects RunView params an external data source can't honor, rather than silently dropping * them. Most important is AfterKey (keyset pagination): the external read path only supports * offset paging, so a silently-dropped AfterKey would return the same page on every call — an * infinite loop / duplicate processing in deep-pagination jobs. Aggregates and a non-empty * UserSearchString likewise can't be evaluated remotely. Throws a clear error naming the param. */ protected assertExternalRunViewParamsSupported(params: RunViewParams, entityName: string): void; /** * Folds a saved view's stored WhereClause and OrderByClause into the RunView params before * external dispatch. The external branch returns before the normal SQL path that applies * them, so without this a UserView over an external entity would silently return unfiltered, * unordered rows. The view's WhereClause is ANDed with any caller ExtraFilter; the view's * OrderByClause is used only when the caller supplied no OrderBy. Returns params unchanged * when there is no saved view. */ protected mergeExternalViewParams(params: RunViewParams, viewEntity: MJUserViewEntityExtended | null, user: UserInfo): Promise; /** * Checks the server's LocalCacheManager for cached data matching the client's request. * Returns the resolution if found (either 'current' or server-cached data to serve), * or null if the server cache doesn't have this data. */ private resolveFromServerCache; /** * Packages server-cached data as a RunViewWithCacheCheckResult for return to the client. */ private serveFromServerCache; /** * Runs a differential query and returns only changes since the client's cached state. * Includes updated/created rows and deleted record IDs. * Falls back to full query if hidden deletes are detected. */ protected runDifferentialQueryAndReturn(params: RunViewParams, entityInfo: EntityInfo, clientMaxUpdatedAt: string, clientRowCount: number, serverStatus: { maxUpdatedAt?: string; rowCount?: number; }, whereSQL: string, viewIndex: number, contextUser?: UserInfo, callerFields?: string[] | null): Promise>; /** * Gets IDs of records deleted since a given timestamp. * Uses dialect-neutral quoting. Subclasses can override for parameterized queries. */ protected getDeletedRecordIDsSince(entityID: string, sinceTimestamp: string, contextUser?: UserInfo): Promise; /** * Gets rows updated/created since a given timestamp. * Uses dialect-neutral quoting and TransformExternalSQLClause for OrderBy. */ protected getUpdatedRowsSince(params: RunViewParams, entityInfo: EntityInfo, sinceTimestamp: string, whereSQL: string, contextUser?: UserInfo): Promise; /**************************************************************************/ /**************************************************************************/ /** * Smart cache validation for batch RunQueries. * For each query, if cacheStatus is provided, checks CacheValidationSQL to determine staleness. * Returns 'current' if cache is valid, 'stale' with fresh data, or 'no_validation' if * the query has no CacheValidationSQL configured. */ RunQueriesWithCacheCheck(params: RunQueryWithCacheCheckParams[], contextUser?: UserInfo): Promise>; /** * B45 override of the RunQuery cache-serve seam — enforce the SAME authorization on a * cache HIT that the miss path enforces. Resolution uses `resolveQuery()` (QueryEngine as * the single source of truth, with CategoryPath disambiguation of same-named queries — the * B46 pairing), and authorization is the FULL `MJQueryEntityExtended.UserCanRun` (roles + * entity CanRead + recursive composition) — exactly the check `ValidateQueryForExecution` * applies before a real execution. The base's roles-only check would let a user read cached * rows of a query whose underlying entities they cannot read. * * Non-throwing by contract: any resolution error degrades to `resolvable: false`, which the * gate treats as "fall through to authorized execution" — one extra query run, never an * unauthorized serve and never a crashed cache path. */ protected ResolveQueryCacheAuthorization(params: RunQueryParams, user?: UserInfo): QueryCacheAuthorization; /** * Resolves a query from RunQueryParams (by ID or Name+CategoryPath). * Uses QueryEngine as the single source of truth for query metadata. */ protected resolveQuery(params: RunQueryParams): MJQueryEntityExtended | undefined; /** * Validates that a query can be executed by the given user. Checks both permissions * and approval status. Permission failures throw an error. Non-approved status * emits a console warning but allows execution to proceed, enabling query testing * before formal approval. * * @param query - The resolved MJQueryEntityExtended to validate * @param contextUser - The user attempting to execute the query * @throws Error if the user does not have permission to run the query */ protected ValidateQueryForExecution(query: MJQueryEntityExtended, contextUser?: UserInfo): void; /** * Resolves a category path string to a QueryCategoryInfo ID. */ protected resolveCategoryPath(categoryPath: string): string | null; /** * Executes cache status checks for multiple queries using their CacheValidationSQL. * Default: parallel individual queries. SQL Server overrides for batch execution. */ protected getBatchedQueryCacheStatus(items: Array<{ index: number; item: RunQueryWithCacheCheckParams; query: MJQueryEntityExtended; }>, contextUser?: UserInfo): Promise>; /** * Runs a full query and returns results with cache metadata. */ protected runFullQueryAndReturnForQuery(params: RunQueryParams, queryIndex: number, status: 'stale' | 'no_validation', contextUser?: UserInfo, queryId?: string, validationStamp?: { maxUpdatedAt?: string; rowCount?: number; }): Promise>; /**************************************************************************/ /**************************************************************************/ /** * Full query execution pipeline: resolve → validate → compose → template → cache check → * execute → paginate → audit → cache store. Platform providers inherit this; only * `ExecuteSQL()` is platform-specific. */ protected InternalRunQuery(params: RunQueryParams, contextUser?: UserInfo): Promise; /** * Batch query execution — runs all queries in parallel. */ protected InternalRunQueries(params: RunQueryParams[], contextUser?: UserInfo): Promise; /** * Executes an ad-hoc SQL query directly, with security validation. * SQL must be a SELECT or WITH (CTE) statement — mutations are rejected. */ protected ExecuteAdhocQuery(params: RunQueryParams, contextUser?: UserInfo): Promise; /** * Finds a query from RunQueryParams and validates user permissions. * Uses `resolveQuery()` for lookup and `ValidateQueryForExecution()` for permissions. */ /** * A saved query is "external" when it resolves to a Query bound to an external data source. * Used by the base RunQuery CacheLocal layer to defer external-query caching to * InternalRunQuery's runExternalQueryWithCache. Non-throwing — resolves from cached metadata. */ protected IsExternalQuery(params: RunQueryParams): boolean; protected findAndValidateQuery(params: RunQueryParams, contextUser?: UserInfo): MJQueryEntityExtended; /** * Processes query parameters: resolves `{{query:"..."}}` composition tokens, * then applies Nunjucks template substitution for `{{param}}` tokens. * * Delegates to {@link RenderPipeline.Run} for the composition → template * pipeline. Paging is handled separately by the caller (InternalRunQuery). */ protected processQueryParameters(query: MJQueryEntityExtended, parameters?: Record, contextUser?: UserInfo): { finalSQL: string; appliedParameters: Record; renderResult: RenderResult; }; /** * Checks the columns an external-data-source query returned against the fields the Query * *declares* (its QueryField metadata) and logs a warning naming any declared field that * is missing — the signature of a remote object whose columns have drifted (renamed or * dropped). * * Per the External Data Sources plan this is intentionally NON-FATAL: the rows are still * returned unchanged. Drift is surfaced for diagnosis, not treated as a hard failure — * the declared field metadata can legitimately lag the remote schema, and the plan calls * for "a warning logged, rows still returned." Matching is case-insensitive because remote * dialects differ in column casing (e.g. Snowflake uppercases unquoted identifiers). No-op * when the query declares no fields or returned no rows (columns can't be inspected without * a row, and an empty result is legitimate). Extra columns beyond the declared set are fine. */ /** * Executes an external-data-source query with TTL-based result caching keyed off the data * source's DefaultCacheTTLSeconds. External queries can't be event-invalidated (their data * lives on a remote system), so a time-bounded cache is the read-cost mitigation the plan * calls for — notably for warehouses like Snowflake. The data source's TTL is the source of * truth (a TTL of <= 0 disables caching for the source); field-drift warnings still apply to * freshly-fetched rows. Ad-hoc SQL (params.SQL) is never cached. */ protected runExternalQueryWithCache(query: MJQueryEntityExtended, finalSQL: string, params: RunQueryParams, router: ExternalDataSourceReadRouter, contextUser?: UserInfo): Promise; protected warnIfExternalQueryFieldsMissing(query: MJQueryEntityExtended, result: RunQueryResult): RunQueryResult; /** * Lower-layer execution: resolves composition, processes templates, executes SQL. * This is the single execution pathway used by both saved queries (via RunQuery upper layer) * and transient test queries (via TestQuerySQL resolver). * * Processing order: * 1. Resolve {{query:"..."}} composition tokens → CTEs (inline deps checked first, then Metadata) * 2. Process {{ param }} Nunjucks templates (if UsesTemplate or any dependency uses templates) * 3. Apply MaxRows safety limit (wrap with TOP/LIMIT if specified) * 4. Execute the fully resolved SQL * 5. Return results with execution metadata */ protected InternalExecuteQueryFromSpec(spec: QueryExecutionSpec, contextUser?: UserInfo): Promise; /** * Resolves composition tokens and Nunjucks templates for a QueryExecutionSpec. * * Delegates to {@link RenderPipeline.Run} for the composition → template * pipeline. Supports inline dependencies for transient query testing. */ private resolveSpecParameters; /** * Checks the query cache for existing results and returns them if valid. * Currently always returns null (query caching is not active). */ protected checkQueryCache(_query: MJQueryEntityExtended, _params: RunQueryParams, _appliedParameters: Record): Promise; /** * Checks the paged cache for a specific page of query results. * Returns a full RunQueryResult on hit, null on miss. * Currently always returns null (query caching is not active). */ protected checkPagedQueryCache(_query: MJQueryEntityExtended, _params: RunQueryParams, _appliedParameters: Record): Promise; /** * Executes the query SQL and tracks execution time. */ protected executeQueryWithTiming(sql: string, contextUser?: UserInfo): Promise<{ result: Record[]; executionTime: number; }>; /** * Applies in-memory pagination to query results based on StartRow and MaxRows parameters. */ protected applyQueryPagination(results: Record[], params: RunQueryParams): { paginatedResult: Record[]; totalRowCount: number; }; /** * Optional, additive post-query enrichment step. Resolves the {@link QueryResultEnricherBase} * registered under `params.Enrichment.EnricherKey` via the MJGlobal ClassFactory and awaits * it on the result rows, returning whatever (column-appended) rows it produces. * * Fully decoupled + resilient by design: * - **Decoupled**: the enricher is resolved by string key through the ClassFactory, so this * provider takes no static dependency on any concrete enricher (e.g. Predictive Studio's * ML-scoring enricher). When no enricher is registered under the key — i.e. the providing * package isn't loaded — {@link resolveQueryResultEnricher} returns `null` and we no-op, * returning the original rows. * - **Resilient**: the whole call is wrapped in try/catch. On ANY failure (resolution, * scorer error, bad config) we LogError and return the ORIGINAL un-enriched rows. A * scoring problem must NEVER break the underlying query. * * The loaded {@link QueryInfo} (when resolvable from the executed query's id) is passed * through so an enricher can read the query's associated entity/fields. * * @param rows the assembled, paginated result rows to enrich * @param params the run params carrying the {@link RunQueryEnrichment} directive * @param query the executed query entity (used to resolve its {@link QueryInfo} metadata) * @param contextUser the request user, threaded through for isolation/audit * @returns the enriched rows on success, or the original rows on any failure / no-op */ protected enrichQueryResults(rows: Record[], params: RunQueryParams, query: MJQueryEntityExtended, contextUser?: UserInfo): Promise[]>; /** * Creates an audit log record for query execution (fire-and-forget). * Only logs if the query has `AuditQueryRuns` enabled or `ForceAuditLog` is set. */ protected auditQueryExecution(query: MJQueryEntityExtended, params: RunQueryParams, finalSQL: string, rowCount: number, totalRowCount: number, executionTime: number, contextUser?: UserInfo): void; /** * Caches query results if caching is enabled for the query. * Currently a no-op (query caching is not active). */ protected cacheQueryResults(_query: MJQueryEntityExtended, _parameters: Record, _results: Record[]): Promise; /**************************************************************************/ /**************************************************************************/ /** * Loads a single entity record by composite key, with optional relationship loading. * Uses dialect-neutral quoting for all SQL construction. */ Load(entity: BaseEntity, compositeKey: CompositeKey, entityRelationshipsToLoad: string[] | null | undefined, user: UserInfo): Promise | null>; /**************************************************************************/ /**************************************************************************/ /** * Checks whether an existing record passes the RLS filter for a given permission type. * Executes: SELECT COUNT(*) AS cnt FROM view WHERE PK=value AND (RLS filter) * Returns true if the record matches (cnt > 0), false otherwise. */ protected CheckRecordRLS(entity: BaseEntity, user: UserInfo, type: EntityPermissionType): Promise; /** * Checks whether a new record's field values pass the Create RLS filter. * Builds a synthetic single-row subquery from entity field values, then tests the RLS filter against it. */ protected CheckCreateRLS(entity: BaseEntity, user: UserInfo): Promise; /** * Builds field projections for the Create RLS synthetic row subquery. * Only includes non-virtual fields that have non-null values. */ private BuildCreateRLSProjections; /**************************************************************************/ /**************************************************************************/ /** * Builds a parameter placeholder for parameterized queries. * Default: PG-style ($1, $2, ...). SQL Server overrides to @p0, @p1, etc. */ BuildParameterPlaceholder(index: number): string; /** * Retrieves a dataset by name, executing all item queries via ExecuteSQLBatch * and aggregating results. Uses dialect-neutral quoting for all SQL construction. * * ExecuteSQLBatch gives SQL Server true multi-result-set batching automatically, * while PG (and the default) use parallel individual queries. */ GetDatasetByName(datasetName: string, itemFilters?: DatasetItemFilterType[], contextUser?: UserInfo, providerToUse?: IMetadataProvider, forceRefresh?: boolean): Promise; /**************************************************************************/ /**************************************************************************/ /** * Retrieves status information for a dataset by name: per-entity row count and * latest update date. Uses ExecuteSQLBatch for per-item status queries. */ GetDatasetStatusByName(datasetName: string, itemFilters?: DatasetItemFilterType[], contextUser?: UserInfo, providerToUse?: IMetadataProvider): Promise; /**************************************************************************/ /**************************************************************************/ /** * Validates columns for a dataset item and returns the column list string. * Returns null if columns are invalid. */ protected getColumnsForDatasetItem(item: Record, datasetName: string): string | null; /**************************************************************************/ /**************************************************************************/ /** * Computes the latest update date for a dataset item from its result rows and dataset metadata. * Used by both the cache-hit and cache-miss paths in GetDatasetByName. * @param rows - The result rows (from cache or SQL) * @param dateFieldToCheck - The field name to scan for latest date * @param item - The dataset item metadata row (contains DatasetItemUpdatedAt, DatasetUpdatedAt) * @returns The latest date across all rows and dataset metadata */ protected computeLatestUpdateDate(rows: unknown[], dateFieldToCheck: string, item: Record): Date; /** * Extracts the MAX value of a specified date field from result rows as an ISO string. * Used for write-through caching of dataset item results. * @param rows - The result rows * @param dateFieldToCheck - The field name to scan * @returns ISO string of the max date, or current time if no dates found */ protected extractMaxUpdatedAtFromRows(rows: unknown[], dateFieldToCheck: string): string; /** * Server-side semantic ranking pass for {@link ProviderBase.SearchEntity} * (and, by extension, the batched {@link ProviderBase.SearchEntities}). * * The query embedding MUST be generated with the same model that produced * the indexed vectors — otherwise cosine scores compare apples to oranges * and rankings are garbage. We look up the EntityDocument's `AIModelID` via * `AIEngine.Models` to recover the driver class / APIName and call * `EmbedText(model, text)` directly. If the EntityDocument does not specify * a model (or the model isn't loaded) we fall back to * `EmbedTextLocal` (highest-power local model) only as a last resort. * * Vector ranking runs against the in-process `SimpleVectorServiceProvider`, * which rehydrates the vector pool for `entityDocumentId` from * `MJ: Entity Record Documents.VectorJSON` rows. * * Failures (no embedding model available, vector index miss) degrade to an * empty result set so hybrid mode can still surface lexical matches. */ protected searchEntitiesSemanticPass(entityDocumentId: string, searchText: string, overFetch: number, embeddingAIModelId: string | null, contextUser: UserInfo | undefined): Promise; } //# sourceMappingURL=GenericDatabaseProvider.d.ts.map