/* eslint-disable no-console */ import { FormatHelpers, OutputModel, TS_DESCRIPTION_PRESET, typeScriptDefaultModelNameConstraints, typeScriptDefaultPropertyKeyConstraints, TypeScriptGenerator, } from '@asyncapi/modelina'; import Parser, { AsyncAPIDocumentInterface, fromFile, MessageInterface, } from '@asyncapi/parser'; import endent from 'endent'; import * as fs from 'fs'; import * as path from 'path'; import { exitCode } from '../../common'; import { MessageCodegenOptions } from './message-codegen-options'; import { ADDITIONAL_PROPERTIES_PRESET, ANY_TO_UNKNOWN_PRESET, EXPORT_TYPES_PRESET, IMPORTS_PRESET, NULLABLE_PROPERTY_TO_UNION_PRESET, } from './presets'; import { createDummyReservedKeywordsChecker, getChannelAction, getChannelAggregateIdField, getChannelAggregateType, getMessageTitle, getModelPathPrefix, getPayloadTitle, getRelativeDir, getServiceId, getServiceTitle, removeXParserProperties, toPosixPath, } from './utils'; const AX_COMMON_SERVICE_ID = 'ax-common-service'; /** * Processes AsyncAPI document and generates Typescript classes from it. * Generated output: * - message payloads will be transformed into Typescript types and bundled JSON schemas * - channels information will be used to generate MessagingSettings, that can be used to configure RabbitMQ connection. * @param inputDir - Path to input directory - AsyncAPI document root. * @param filePattern - Regular expression that matches suitable input files. * @param outputDir - Path to output directory - output root. */ export class Codegen { private readonly schemaRoot: string; private readonly filePattern: RegExp; private readonly outputRoot: string; private readonly typesOutputRoot: string; private readonly schemasOutputRoot: string; private readonly messagingSettingsOutputRoot: string; constructor(options: MessageCodegenOptions) { this.schemaRoot = path.resolve(options.inputDir); try { this.filePattern = new RegExp(options.filePattern); } catch (error) { console.error( `${options.filePattern} is not a valid regular expression.`, ); process.exit(1); } this.outputRoot = path.resolve(options.outputDir); this.typesOutputRoot = path.join(this.outputRoot, 'types'); this.schemasOutputRoot = path.join(this.outputRoot, 'schemas'); this.messagingSettingsOutputRoot = path.join(this.outputRoot, 'config'); } public async run(): Promise { console.log('Running message codegen.'); console.log(`* schema root: ${this.schemaRoot}`); console.log(`* output root: ${this.outputRoot}`); try { await this.walk(this.schemaRoot); await this.barrelExportTs( [], [ this.typesOutputRoot, this.schemasOutputRoot, this.messagingSettingsOutputRoot, ], path.join(this.outputRoot, 'index.ts'), ); } catch (e) { console.log(e); process.exit(exitCode); } } /** * Recursively walks a schemas directory and generates TS + bundled JSON schema files. * @param dir - Directory to walk. */ async walk(dir: string): Promise { const items = await fs.promises.readdir(dir); const fullPathItems = items.map((i) => path.join(dir, i)); const dirs = fullPathItems.filter((i) => fs.statSync(i).isDirectory()); const files = fullPathItems.filter( (i) => fs.statSync(i).isFile() && this.filePattern.test(i), ); if ([...files, ...dirs].length === 0) { return; } for (const asyncApiFile of files) { console.log(`Processing ${asyncApiFile}.`); await this.processAsyncAPIDocument(asyncApiFile); } for (const directory of dirs) { await this.walk(directory); } if (files.length === 0) { //Calculate correct TS output related directories for barrel exports. const tsDirs = dirs.map((d) => path.join(this.typesOutputRoot, path.relative(this.schemaRoot, d)), ); const tsIndexOutPath = path.join( this.typesOutputRoot, getRelativeDir(this.schemaRoot, dir), 'index.ts', ); await this.barrelExportTs([], tsDirs, tsIndexOutPath); // Calculate correct JSON schema output related directories for barrel exports. const schemaDirs = dirs.map((d) => path.join(this.schemasOutputRoot, path.relative(this.schemaRoot, d)), ); const schemaIndexOutPath = path.join( this.schemasOutputRoot, getRelativeDir(this.schemaRoot, dir), 'index.ts', ); await this.barrelExportSchema([], schemaDirs, schemaIndexOutPath); // Calculate correct Message Settings output related directories for barrel exports. const settingsDirs = dirs.map((d) => path.join( this.messagingSettingsOutputRoot, path.relative(this.schemaRoot, d), ), ); const messagingIndexOutPath = path.join( this.messagingSettingsOutputRoot, getRelativeDir(this.schemaRoot, dir), 'index.ts', ); await this.barrelExportSettings([], settingsDirs, messagingIndexOutPath); } } /** * Builds a path for the generated TS file from an input JSON schema path, retains directory structure. * @param outputPath - output directory. * @param typeName - Name of the type. */ buildTsOutPath(outputPath: string, typeName: string): string { return path.join(outputPath, `${FormatHelpers.toParamCase(typeName)}.ts`); } /** * Builds a path for the bundled JSON schema file from an input JSON schema path, retains directory structure. * @param outputPath - output directory. * @param typeName - Name of the type. */ buildSchemaOutPath(outputPath: string, typeName: string): string { return path.join(outputPath, `${FormatHelpers.toParamCase(typeName)}.json`); } /** * Generates TS interfaces and JSON schemas from AsyncAPI schema. * @param asyncApiFile - path to AsyncAPI document */ async processAsyncAPIDocument(asyncApiFile: string): Promise { const parser = new Parser(); const { document, diagnostics } = await fromFile( parser, asyncApiFile, ).parse(); if (document === undefined) { console.error(`Failed to parse ${asyncApiFile}.`); console.error(diagnostics); process.exit(1); } //export Typescript models const tsModelsOutputPath = path.join( this.typesOutputRoot, getRelativeDir(this.schemaRoot, asyncApiFile), ); await this.exportTsModels(document, tsModelsOutputPath); //export JSON Schemas const schemasOutputPath = path.join( this.schemasOutputRoot, getRelativeDir(this.schemaRoot, asyncApiFile), ); await this.exportSchemas(document, schemasOutputPath); //export message settings const messagingSettingsOutputPath = path.join( this.messagingSettingsOutputRoot, getRelativeDir(this.schemaRoot, asyncApiFile), ); await this.exportSettings(document, messagingSettingsOutputPath); } /** * Export all TS models from AsyncAPI document * @param asyncAPIDocument - AsyncAPI Document object * @param outputPath - output path for generated files */ async exportTsModels( asyncAPIDocument: AsyncAPIDocumentInterface, outputPath: string, ): Promise { const generator = new TypeScriptGenerator({ modelType: 'interface', enumType: 'union', mapType: 'indexedObject', constraints: { propertyKey: typeScriptDefaultPropertyKeyConstraints({ NAMING_FORMATTER: (name) => FormatHelpers.toSnakeCase(name), NO_RESERVED_KEYWORDS: createDummyReservedKeywordsChecker(), }), modelName: typeScriptDefaultModelNameConstraints({ NO_RESERVED_KEYWORDS: createDummyReservedKeywordsChecker(), }), }, // Order of presets matters! presets: [ NULLABLE_PROPERTY_TO_UNION_PRESET, ANY_TO_UNKNOWN_PRESET, ADDITIONAL_PROPERTIES_PRESET, EXPORT_TYPES_PRESET, TS_DESCRIPTION_PRESET, IMPORTS_PRESET, ], }); let outputModels = await generator.generate(asyncAPIDocument); for (const schema of asyncAPIDocument.components().schemas()) { const referencedModels = await generator.generate(schema.json()); outputModels = outputModels.map( (om) => referencedModels.find((rm) => om.modelName === rm.modelName) || om, ); } const groupedModels: { [key: string]: OutputModel[] } = {}; outputModels.map((outModel) => { const modelPathPrefix = getModelPathPrefix(outModel.modelName); if (groupedModels[modelPathPrefix] === undefined) { groupedModels[modelPathPrefix] = []; } groupedModels[modelPathPrefix].push(outModel); }); const tsDirs: string[] = []; for (const pathPrefix in groupedModels) { const models = groupedModels[pathPrefix]; const groupOutPutPath = path.join(outputPath, pathPrefix); if (pathPrefix) { tsDirs.push(groupOutPutPath); } await fs.promises.mkdir(groupOutPutPath, { recursive: true, }); const tsFiles: string[] = []; for (const outputModel of models) { const outputFilePath = this.buildTsOutPath( groupOutPutPath, outputModel.modelName, ); await fs.promises.writeFile(outputFilePath, outputModel.result); tsFiles.push(outputFilePath); } //generate barrel export for all model in AsyncAPI document const tsIndexOutPath = path.join(groupOutPutPath, 'index.ts'); const existingTsFiles = fs.existsSync(groupOutPutPath) ? (await fs.promises.readdir(groupOutPutPath)) .filter((f) => f.endsWith('.ts') && f !== 'index.ts') .map((f) => path.join(groupOutPutPath, f)) : []; const mergedTsFiles = [...new Set([...tsFiles, ...existingTsFiles])]; await this.barrelExportTs(mergedTsFiles, [], tsIndexOutPath); } if (tsDirs.length > 0) { const tsIndexOutPath = path.join(outputPath, 'index.ts'); const existingDirs = fs.existsSync(outputPath) ? (await fs.promises.readdir(outputPath, { withFileTypes: true })) .filter((d) => d.isDirectory()) .map((d) => path.join(outputPath, d.name)) : []; const mergedDirs = [...new Set([...tsDirs, ...existingDirs])]; await this.barrelExportTs([], mergedDirs, tsIndexOutPath); } } /** * Export all JSON Schemas from AsyncAPI document * @param asyncAPIDocument - AsyncAPI Document object * @param outputPath - output path for generated files */ async exportSchemas( asyncAPIDocument: AsyncAPIDocumentInterface, outputPath: string, ): Promise { const groupedSchemas: { [key: string]: MessageInterface[] } = {}; for (const message of asyncAPIDocument.allMessages()) { const schemaPathPrefix = getModelPathPrefix(getMessageTitle(message)); if (groupedSchemas[schemaPathPrefix] === undefined) { groupedSchemas[schemaPathPrefix] = []; } groupedSchemas[schemaPathPrefix].push(message); } const schemaDirs: string[] = []; for (const pathPrefix in groupedSchemas) { const messages = groupedSchemas[pathPrefix]; const schemaFiles: string[] = []; const groupOutPutPath = path.join(outputPath, pathPrefix); if (pathPrefix) { schemaDirs.push(groupOutPutPath); } await fs.promises.mkdir(groupOutPutPath, { recursive: true, }); for (const msg of messages) { const payload = msg.payload(); if (payload !== undefined && payload.json() !== undefined) { const filteredPayload = removeXParserProperties(payload.json()); const outputSchemaPath = this.buildSchemaOutPath( groupOutPutPath, getMessageTitle(msg), ); await this.bundleSchema(filteredPayload, outputSchemaPath); schemaFiles.push(outputSchemaPath); } } //generate barrel export for all model in AsyncAPI document const schemasIndexOutPath = path.join(groupOutPutPath, 'index.ts'); const existingSchemaFiles = fs.existsSync(groupOutPutPath) ? (await fs.promises.readdir(groupOutPutPath)) .filter((f) => f.endsWith('.json')) .map((f) => path.join(groupOutPutPath, f)) : []; const mergedSchemaFiles = [ ...new Set([...schemaFiles, ...existingSchemaFiles]), ]; await this.barrelExportSchema(mergedSchemaFiles, [], schemasIndexOutPath); } const schemaIndexOutPath = path.join(outputPath, 'index.ts'); const existingSchemaDirs = fs.existsSync(outputPath) ? (await fs.promises.readdir(outputPath, { withFileTypes: true })) .filter((d) => d.isDirectory()) .map((d) => path.join(outputPath, d.name)) : []; const mergedSchemaDirs = [ ...new Set([...schemaDirs, ...existingSchemaDirs]), ]; await this.barrelExportSchema([], mergedSchemaDirs, schemaIndexOutPath); } /** * Export all AsyncAPI document channels to Messaging Settings * @param asyncAPIDocument - AsyncAPI Document object * @param outputPath - output path for generated files */ async exportSettings( asyncAPIDocument: AsyncAPIDocumentInterface, outputPath: string, ): Promise { const serviceTitle = getServiceTitle(asyncAPIDocument); const serviceId = getServiceId(asyncAPIDocument); const channelsData: ChannelData[] = []; for (const channel of asyncAPIDocument.channels()) { const queueName = channel.bindings().get('amqp')?.value().queue.name; const routingKey = channel.address(); if (queueName !== undefined && routingKey != null) { const action = getChannelAction(channel); const payloadName = getPayloadTitle(channel) ?? 'undefined'; const aggregateType = getChannelAggregateType(channel); const aggregateIdField = getChannelAggregateIdField(channel); channelsData.push({ routingKey, queueName, payloadName, acceptedAction: action, isMultiTenant: routingKey.includes('*.*'), aggregateType, aggregateIdField, }); } } await this.generateMessagingSettings( serviceId, serviceTitle, channelsData, outputPath, ); } /** * Generate Messaging Settings based on Channels data. * @param serviceId - service identifier * @param serviceTitle - service title * @param channelsData - AsyncAPI channels information * @param outputPath - output path for generated files */ async generateMessagingSettings( serviceId: string, serviceTitle: string, channelsData: ChannelData[], outputPath: string, ): Promise { const messagingSettingFiles: string[] = []; messagingSettingFiles.push( await this.createMessagingSettingFile( true, serviceId, serviceTitle, channelsData.filter((data) => data.isMultiTenant === true), outputPath, ), ); messagingSettingFiles.push( await this.createMessagingSettingFile( false, serviceId, serviceTitle, channelsData.filter((data) => data.isMultiTenant === false), outputPath, ), ); //barrel export for messaging settings const schemaIndexOutPath = path.join(outputPath, 'index.ts'); await this.barrelExportSettings( messagingSettingFiles.filter((f) => f), [], schemaIndexOutPath, ); } /** * Create new Messaging Settings file. * @param isMultiTenant - is file with multi tenant settings, or not * @param serviceId - service Identifier * @param serviceTitle - service Title * @param channelsData - AsyncAPI channels information * @param outputPath - output path for generated files */ async createMessagingSettingFile( isMultiTenant: boolean, serviceId: string, serviceTitle: string, channelsData: ChannelData[], outputPath: string, ): Promise { if (channelsData.length === 0) { return ``; } const baseClass = isMultiTenant ? `MultiTenantMessagingSettings` : `MessagingSettings`; const className = `${FormatHelpers.toPascalCase(serviceTitle)}${baseClass}`; const action = isMultiTenant ? 'extends' : 'implements'; const visibility = isMultiTenant ? '' : 'public readonly '; const toStringOverride = isMultiTenant ? ' override' : ''; const constructor = serviceId === AX_COMMON_SERVICE_ID ? endent`private constructor( ${visibility}serviceId: string, ${visibility}messageType: string, ${visibility}queue: string, ${visibility}routingKey: string, ${visibility}action: 'command' | 'event', ${visibility}aggregateType: string, ) { ${ isMultiTenant ? `super(serviceId, messageType, queue, routingKey, action, aggregateType);` : '' } }` : endent`private constructor( ${visibility}messageType: string, ${visibility}queue: string, ${visibility}routingKey: string, ${visibility}action: 'command' | 'event', ${visibility}aggregateType: string, ) { ${ isMultiTenant ? `super('${serviceId}', messageType, queue, routingKey, action, aggregateType);` : '' } }`; const properties = channelsData.map((data) => this.createMessagingSettingProperty( data, className, serviceId === AX_COMMON_SERVICE_ID, ), ); const content = endent`import { ${baseClass} } from '@axinom/mosaic-message-bus-abstractions'; export class ${className} ${action} ${baseClass} { ${properties.join(`\n`)} ${ isMultiTenant || serviceId === AX_COMMON_SERVICE_ID ? `` : `\npublic readonly serviceId = '${serviceId}';\n` } ${constructor} public${toStringOverride} toString = (): string => { return this.messageType; }; }`; const filePath = path.join( outputPath, `${FormatHelpers.toParamCase(className)}.ts`, ); await fs.promises.mkdir(path.dirname(filePath), { recursive: true, }); await fs.promises.writeFile(filePath, content); return filePath; } /** * Create Messaging Settings property from Channel Data. * @param channelData - AsyncAPI channel information * @param settingsClassName - class name of messaging setting * @param isCommonService - flag to indicate if it's for `AX_COMMON_SERVICE_ID` */ createMessagingSettingProperty( channelData: ChannelData, settingsClassName: string, isCommonService: boolean, ): string { const messageType = FormatHelpers.toPascalCase( channelData.payloadName.replace('command', '').replace('event', ''), ); const action = channelData.acceptedAction === 'publish' ? 'command' : 'event'; const contain = action === 'command' ? 'must contain' : 'contains'; let aggregateIdFieldDesc = ''; switch (channelData.aggregateIdField) { case 'UNDEFINED_ID': aggregateIdFieldDesc = `The aggregate ID is not known (yet) so the field ${contain} the value "UNDEFINED_ID".`; break; case 'MULTIPLE_IDS': aggregateIdFieldDesc = `There are multiple aggregate IDs in the payload so the field ${contain} the value "MULTIPLE_IDS".`; break; default: aggregateIdFieldDesc = `The aggregate ID field ${contain} the value of the "${channelData.aggregateIdField}" field.`; break; } if (isCommonService) { return endent` /** * Defines the messaging settings for the ${action} with message type * "${messageType}" and aggregate type "${channelData.aggregateType}". * ${aggregateIdFieldDesc} */ public static Get${messageType}Settings(serviceId: string) { return new ${settingsClassName}( serviceId, '${messageType}', '${channelData.queueName}', '${channelData.routingKey}', '${action}', '${channelData.aggregateType}' ); }`; } else { return endent` /** * Defines the messaging settings for the ${action} with message type * "${messageType}" and aggregate type "${channelData.aggregateType}". * ${aggregateIdFieldDesc} */ public static ${messageType} = new ${settingsClassName}( '${messageType}', '${channelData.queueName}', '${channelData.routingKey}', '${action}', '${channelData.aggregateType}' );`; } } /** * Bundles a JSON schema into a self-contained file by including all external refs. * @param jsonSchema - Schema object. * @param outPath - Output JSON schema path. */ async bundleSchema(jsonSchema: unknown, outPath: string): Promise { await fs.promises.mkdir(path.dirname(outPath), { recursive: true }); await fs.promises.writeFile(outPath, JSON.stringify(jsonSchema, null, 2)); } // TODO: Consider doing named exports based on message groups to not just put all messages in one namespace. // TODO: Maybe some templating engine would be better than just building strings. /** * Generates a barrel index.ts for all modules inside `outPath`. * In addition it generates a schemas enum for the included modules to make it easier to map them to bundled JSON schemas. * @param files - Files to roll up. * @param dirs - Directories to roll up. * @param outPath - Path where to write `index.ts`. */ async barrelExportTs( files: string[], dirs: string[], outPath: string, ): Promise { console.log(`Rolling up TS exports to ${outPath}.`); const items = [ ...files.map((f) => path.basename(f, '.ts')), ...dirs.filter((d) => fs.existsSync(d)).map((d) => path.basename(d)), ]; if (items.length < 1) { return; } const exports = `${items .sort() .map((p) => `export * from './${p}';`) .join('\n')}`; let schemaEnum = ''; let typeNamesEnum = ''; // TODO: Break this up into smaller pieces. // TODO: Consider adding docstring to generated enums. if (files.length > 0) { const sortedFiles = files.sort(); // TODO: Message envelope is handled separately, we could remove this entirely. // If message-envelope requires some special handling. if (files.length === 1 && files[0].endsWith('message-envelope.ts')) { const file = files[0]; schemaEnum = endent` export enum MessageEnvelopeSchema { MessageEnvelope = '${toPosixPath(this.tsPathToSchemaPath(file))}' } `; } else { let enumNameBase = FormatHelpers.toPascalCase( getRelativeDir(this.typesOutputRoot, path.dirname(outPath)), ); if (!enumNameBase.includes('Types')) { enumNameBase = enumNameBase.replace('Payloads', ''); // Remove the Payloads prefix to reduce noise schemaEnum = endent` export enum ${enumNameBase}Schemas { ${sortedFiles .map( (f) => `${FormatHelpers.toPascalCase( path.basename(f, '.ts'), )} = '${toPosixPath(this.tsPathToSchemaPath(f))}'`, ) .join(',\n')} }`; typeNamesEnum = endent` export enum ${enumNameBase}Types { ${sortedFiles .sort() .map( (f) => `${FormatHelpers.toPascalCase( path.basename(f, '.ts'), )} = '${FormatHelpers.toPascalCase(path.basename(f, '.ts'))}'`, ) .join(',\n')} }`; } } } const contents = endent` ${exports} ${schemaEnum} ${typeNamesEnum} `; await fs.promises.writeFile(outPath, contents); } /** * Generates a barrel index.ts for all Messaging Settings inside `outPath`. * @param files - Files to roll up. * @param dirs - Directories to roll up. * @param outPath - Path where to write `index.ts`. */ async barrelExportSettings( files: string[], dirs: string[], outPath: string, ): Promise { console.log(`Rolling up Messaging Settings exports to ${outPath}.`); const items = [ ...files.map((f) => path.basename(f, '.ts')), ...dirs.filter((d) => fs.existsSync(d)).map((d) => path.basename(d)), ]; const dirExports = dirs .sort() .filter((d) => fs.existsSync(d)) .map((d) => path.basename(d)); if (items.length < 1) { return; } let contents = ''; if (files.length > 0) { const baseNames = files.map((f) => path.basename(f, '.ts')).sort(); const messagingImports = baseNames.map((n) => `export * from './${n}';`); contents = endent` ${messagingImports.join('\n')} `; } contents += endent`\n ${dirExports.map((d) => `export * from './${d}';`).join('\n')}`; await fs.promises.writeFile(outPath, contents); } /** * Generates a barrel index.ts for all JSON schemas inside `outPath`. * @param files - Files to roll up. * @param dirs - Directories to roll up. * @param outPath - Path where to write `index.ts`. */ async barrelExportSchema( files: string[], dirs: string[], outPath: string, ): Promise { console.log(`Rolling up JSON exports to ${outPath}.`); const items = [ ...files.map((f) => path.basename(f, '.json')), ...dirs.filter((d) => fs.existsSync(d)).map((d) => path.basename(d)), ]; const dirExports = dirs .sort() .filter((d) => fs.existsSync(d)) .map((d) => path.basename(d)); if (items.length < 1) { return; } let contents = ''; if (files.length > 0) { const baseNames = files.map((f) => path.basename(f, '.json')).sort(); const schemaImports = baseNames.map( (n) => `import * as ${FormatHelpers.toPascalCase(n)} from './${n}.json';`, ); const schemaExports = baseNames.map( (n) => `export const ${FormatHelpers.toPascalCase( n, )}Schema = ${FormatHelpers.toPascalCase(n)};`, ); contents = endent` ${schemaImports.join('\n')} ${schemaExports.join('\n')} `; } contents += endent`\n ${dirExports.map((d) => `export * from './${d}';`).join('\n')}`; await fs.promises.writeFile(outPath, contents); } /** * Converts a generated TS path to a corresponding JSON schema path. * @param tsFile - Path to a generated TS file. */ tsPathToSchemaPath(tsFile: string): string { return path.join( getRelativeDir(this.typesOutputRoot, tsFile), `${path.basename(tsFile, '.ts')}.json`, ); } } /** * Metadata of channels for settings generation. */ interface ChannelData { /** Channel routing key */ routingKey: string; /** Channel queue */ queueName: string; /** Channel model */ payloadName: string; /** Accepted Action*/ acceptedAction: 'subscribe' | 'publish'; /** Channel multi tenancy */ isMultiTenant: boolean; /** Aggregate root type of the channel message */ aggregateType: string; /** * Aggregate root ID field of the channel message or "UNDEFINED_ID" if no ID * is known for this message or "MULTIPLE_IDS" if the message contains * multiple IDs. */ aggregateIdField: string; }