import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { api, ParseSyncConfigOptions, ReplicationHeadCallback } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import { ErrorCode, logger } from '@powersync/lib-services-framework'; import { CheckpointImplementation } from '../replication/checkpoints/CheckpointImplementation.js'; import { createCheckpointImplementation } from '../replication/checkpoints/create-checkpoint-implementation.js'; import { MongoManager } from '../replication/MongoManager.js'; import { CHECKPOINTS_COLLECTION, detectDocumentDb } from '../replication/replication-utils.js'; import * as types from '../types/types.js'; import { escapeRegExp } from '../utils.js'; import { inferCollectionSchema, isSchemaInferenceFailure } from './infer-collection-schema.js'; const SCHEMA_INFERENCE_CONCURRENCY = 4; export class MongoRouteAPIAdapter implements api.RouteAPI { protected client: mongo.MongoClient; public db: mongo.Db; connectionTag: string; defaultSchema: string; private checkpointImplementation: CheckpointImplementation | null = null; private documentDbDetection: Promise | null = null; constructor(protected config: types.ResolvedConnectionConfig) { const manager = new MongoManager(config); this.client = manager.client; this.db = manager.db; this.defaultSchema = manager.db.databaseName; this.connectionTag = config.tag ?? sync_rules.DEFAULT_TAG; } getParseSyncRulesOptions(): ParseSyncConfigOptions { return { defaultSchema: this.defaultSchema, connections: { [this.connectionTag]: { type: this.config.type } } }; } async validateSourceCapabilities(config: sync_rules.SyncConfig): Promise { return this.validateMongoFilters(config); } /** * Validate MongoDB pre-filtering support. External modules can override this method to account for * their plan parameters and filter capabilities while retaining other source-capability checks. */ protected async validateMongoFilters(config: sync_rules.SyncConfig): Promise { const filtered = sync_rules.hasMongoFilterExpressions( config instanceof sync_rules.PrecompiledSyncConfig ? (config.plan.sourceTableConfig ?? {}) : {}, this.connectionTag ); return filtered ? [ { level: 'fatal', message: `[${ErrorCode.PSYNC_S1348}] MongoDB replication pre-filtering is not supported in this edition of the PowerSync service.` } ] : []; } async shutdown(): Promise { await this.client.close(); } async [Symbol.asyncDispose]() { await this.shutdown(); } async getSourceConfig(): Promise { return this.config; } async getConnectionStatus(): Promise { const base = { id: this.config.id, uri: lib_mongo.baseUri(this.config) }; try { await this.client.connect(); await this.db.command({ hello: 1 }); } catch (e) { return { ...base, connected: false, errors: [{ level: 'fatal', message: e.message }] }; } return { ...base, connected: true, errors: [] }; } async executeQuery(query: string, params: any[]): Promise { return service_types.internal_routes.ExecuteSqlResponse.encode({ results: { columns: [], rows: [] }, success: false, error: 'SQL querying is not supported for MongoDB' }); } async getDebugTablesInfo( tablePatterns: sync_rules.TablePattern[], sqlSyncRules: sync_rules.SyncConfig ): Promise { let result: api.PatternResult[] = []; const validatePostImages = (schema: string, collection: mongo.CollectionInfo): service_types.ReplicationError[] => { if (this.config.postImages == types.PostImagesOption.OFF) { return []; } else if (!collection.options?.changeStreamPreAndPostImages?.enabled) { if (this.config.postImages == types.PostImagesOption.READ_ONLY) { return [ { level: 'fatal', message: `changeStreamPreAndPostImages not enabled on ${schema}.${collection.name}` } ]; } else { return [ { level: 'warning', message: `changeStreamPreAndPostImages not enabled on ${schema}.${collection.name}, will be enabled automatically` } ]; } } else { return []; } }; for (let tablePattern of tablePatterns) { const schema = tablePattern.schema; let patternResult: api.PatternResult = { schema: schema, pattern: tablePattern.tablePattern, wildcard: tablePattern.isWildcard }; result.push(patternResult); let nameFilter: RegExp | string; if (tablePattern.isWildcard) { nameFilter = new RegExp('^' + escapeRegExp(tablePattern.tablePrefix)); } else { nameFilter = tablePattern.name; } // Check if the collection exists const collections = await this.client .db(schema) .listCollections( { name: nameFilter }, { nameOnly: false } ) .toArray(); if (tablePattern.isWildcard) { patternResult.tables = []; for (let collection of collections) { const ref: sync_rules.SourceTableRef = { connectionTag: this.connectionTag, schema, name: collection.name }; let errors: service_types.ReplicationError[] = []; if (collection.type == 'view') { errors.push({ level: 'warning', message: `Collection ${schema}.${collection.name} is a view` }); } else { errors.push(...validatePostImages(schema, collection)); } const syncData = sqlSyncRules.tableSyncsData(ref); const syncParameters = sqlSyncRules.tableSyncsParameters(ref); patternResult.tables.push({ schema, name: collection.name, replication_id: ['_id'], data_queries: syncData, parameter_queries: syncParameters, errors: errors }); } } else { const ref: sync_rules.SourceTableRef = { connectionTag: this.connectionTag, schema, name: tablePattern.name }; const syncData = sqlSyncRules.tableSyncsData(ref); const syncParameters = sqlSyncRules.tableSyncsParameters(ref); const collection = collections[0]; let errors: service_types.ReplicationError[] = []; if (collections.length != 1) { errors.push({ level: 'warning', message: `Collection ${schema}.${tablePattern.name} not found` }); } else if (collection.type == 'view') { errors.push({ level: 'warning', message: `Collection ${schema}.${tablePattern.name} is a view` }); } else if (!collection.options?.changeStreamPreAndPostImages?.enabled) { errors.push(...validatePostImages(schema, collection)); } patternResult.table = { schema, name: tablePattern.name, replication_id: ['_id'], data_queries: syncData, parameter_queries: syncParameters, errors }; } } return result; } async getReplicationLagBytes(options: api.ReplicationLagOptions): Promise { // There is no fast way to get replication lag in bytes in MongoDB. // We can get replication lag in seconds, but need a different API for that. return undefined; } async createReplicationHead(callback: ReplicationHeadCallback): Promise { const checkpointImplementation = await this.getCheckpointImplementation(); return checkpointImplementation.createReplicationHead(callback); } private isDocumentDb(): Promise { // Share in-flight detection and cache its result for this adapter's lifetime. return (this.documentDbDetection ??= detectDocumentDb(this.db).catch((error) => { // Allow a later request to retry after a transient connection error. this.documentDbDetection = null; throw error; })); } private async getCheckpointImplementation(): Promise { if (this.checkpointImplementation == null) { const isDocumentDb = await this.isDocumentDb(); this.checkpointImplementation = createCheckpointImplementation(isDocumentDb, { client: this.client, db: this.db, // The adapter never streams, so it has no real barrier document. This // random id is only safe because the adapter only ever calls // createReplicationHead, which produces a standalone (stream_id = null) // head that every real ChangeStream observes. It must NOT be used for a // batch barrier (createBatchCheckpoint stamps this id), or the head would // become an own-barrier of a phantom stream that nothing resolves. checkpointStreamId: new mongo.ObjectId(), logger }); } return this.checkpointImplementation; } async getConnectionSchema(): Promise { const isDocumentDb = await this.isDocumentDb(); const databases = await this.db.admin().listDatabases({ nameOnly: true }); const filteredDatabases = databases.databases.filter((db) => { return !['local', 'admin', 'config'].includes(db.name); }); const databaseSchemas: service_types.DatabaseSchema[] = []; for (const db of filteredDatabases) { /** * Filtering the list of database with `authorizedDatabases: true` * does not produce the full list of databases under some circumstances. * This catches any potential auth errors. */ let collections: mongo.CollectionInfo[]; try { collections = await this.client.db(db.name).listCollections().toArray(); } catch (e) { if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') { // Ignore databases we're not authorized to query continue; } throw e; } const tables: (service_types.TableSchema | undefined)[] = new Array(collections.length); const pendingCollections = collections.entries(); let failed = false; const inferCollections = async () => { // Reading from the iterator automatically manages the queue for (const [index, collection] of pendingCollections) { if (failed) { return; } if ([CHECKPOINTS_COLLECTION].includes(collection.name)) { continue; } if (collection.name.startsWith('system.')) { // system.views, system.js, system.profile, system.buckets // https://www.mongodb.com/docs/manual/reference/system-collections/ continue; } if (collection.type == 'view') { continue; } try { const columns = await inferCollectionSchema( this.client.db(db.name).collection(collection.name), isDocumentDb ); // Preserve collection order even when queries finish out of order. tables[index] = { name: collection.name, columns }; } catch (e) { if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') { // Ignore collections we're not authorized to query continue; } if (isSchemaInferenceFailure(e)) { // We know the collection is there, we just could not read its fields. Report it // without columns rather than failing: one collection we cannot sample should not // block validation and deployment for every sync config on the instance. Queries // naming its columns warn about unknown columns; the result is indistinguishable // from an empty collection, so a `SELECT *` query still validates cleanly. logger.warn(`Could not infer the schema of ${db.name}.${collection.name}`, e); tables[index] = { name: collection.name, columns: [] }; continue; } // Fail the whole request on unexpected errors: the response cannot indicate // partial results, so omitting a collection would make an incomplete schema // appear complete and could produce an incorrect generated client schema. failed = true; throw e; } } }; // Each worker holds only schema metadata, with a bounded number of source queries. const workers = Array.from( { length: Math.min(SCHEMA_INFERENCE_CONCURRENCY, collections.length) }, inferCollections ); try { await Promise.all(workers); } catch (e) { // Finish in-flight queries before rejecting the request. Workers stop taking // new collections as soon as one encounters a non-authorization error. await Promise.allSettled(workers); throw e; } databaseSchemas.push({ name: db.name, tables: tables.filter((table) => table != null) }); } return databaseSchemas; } }