import { Plugin, Shape, type Diagnostics, type Transform, type Result, SourceFileShape, StringShape, type Source, type Project, type RecordContext, type Compiler, type Package, type Record, } from '@servicenow/sdk-build-core' import { NowConfig, path as pathModule, ts, type FileSystem, type Logger } from '@servicenow/sdk-build-core' import { RepackService } from '../repack' import { SBOMBuilder } from './sbom-builder' import isEqual from 'lodash/isEqual' import { INVALID_XML_CHARACTERS, applyPathMappings } from '../utils' import zip from 'lodash/zip' const GLUE_CODE_PREFIX = '// @fluent-module' const GLUE_CODE_META_REGEX = new RegExp(`^${GLUE_CODE_PREFIX} (.*);(true|false);(.*)`) // name;isDefault;path const GLUE_CODE_WARNING = ` // WARNING: This code is generated by the ServiceNow SDK in order to provide // support for modular JavaScript. Modifications of any kind are likely to // result in unintended behavior. In most cases, you should edit the source // file of the module imported below. If you are absolutely certain you want // to take control of this code, you can remove this comment to prevent the // SDK from regenerating it. However, you will then be responsible for the // management of this code in your Fluent file.` export const NODE_MODULES = 'node_modules' /** * Check if a module matches any trusted module patterns. * Supports exact matches ('lodash', '@servicenow/sdk') and org wildcards ('@servicenow/*'). * Invalid patterns (like '*' or 'lodash-*-utils') are ignored. */ function isModuleTrusted(moduleName: string, trustedModules: string[], logger: Logger): boolean { if (trustedModules.length === 0) { return false } return trustedModules.some((pattern) => { // @org/* pattern if (pattern.startsWith('@') && pattern.endsWith('/*')) { return moduleName.startsWith(pattern.slice(0, -1)) } // Exact match (no wildcards allowed) if (!pattern.includes('*')) { return moduleName === pattern } // Invalid pattern, ignore logger.warn(`Invalid trusted module pattern used in fluent config: ${pattern}`) return false }) } type GlueCodeMeta = { name: string path: string isDefault: boolean relativePath?: string } function isGlueCode(string: StringShape | string): boolean { return string.startsWith(GLUE_CODE_PREFIX) } function parseMeta(string: StringShape | string): GlueCodeMeta { const value = typeof string === 'string' ? string : string.getValue() const result = value.match(GLUE_CODE_META_REGEX) const [, name, isDefault, path] = result ?? [] if (typeof name !== 'string' || typeof isDefault !== 'string' || typeof path !== 'string') { throw new Error(`Invalid glue code format: ${value}`) } return { name, path, isDefault: isDefault === 'true' } } export class ModuleFunctionShape extends Shape { constructor( source: Source, private readonly meta: GlueCodeMeta ) { super({ source }) } override toString( callExpressionProvider: (functionName: string) => string = (name) => `${name}()`, defaultParams?: string[] ): StringShape { const { name, path, isDefault, relativePath } = this.meta const moduleParamNames = getFunctionParameters(this.getSource()) const variables = zip(defaultParams ?? [], moduleParamNames).map( ([defaultParam, moduleParamName]) => defaultParam ?? moduleParamName ) const moduleCallExpression = callExpressionProvider(name).replace('{{PARAMS}}', variables.join(', ')) const requirePath = relativePath ?? path const code = isDefault ? `const ${name} = require('${requirePath}').default;\n${moduleCallExpression};` : `const { ${name} } = require('${requirePath}');\n${moduleCallExpression};` return Shape.from(this, `${GLUE_CODE_PREFIX} ${name};${isDefault};${path}${GLUE_CODE_WARNING}\n${code}`) .asString() .withContentType('cdata') } override equals(other: unknown): boolean { if (other instanceof ModuleFunctionShape) { return isEqual(this.meta, other.meta) } const string = other instanceof StringShape ? other.getValue() : other if (typeof string === 'string') { if (!isGlueCode(string)) { return false } return isEqual(this.meta, parseMeta(string)) } return super.equals(other) } } /** * Removes invalid control characters that would cause XML parsing errors. */ function sanitizeModuleContent(content: string): string { return content.replace(INVALID_XML_CHARACTERS, '') } function getEmitOutput( file: ts.SourceFile, { fs, diagnostics, config, project, compiler, }: { fs: FileSystem diagnostics: Diagnostics config: NowConfig project: Project compiler: Compiler } ): string | undefined { const path = file.getFilePath() if (!path.endsWith('.ts')) { return file.getFullText() } const tsConfigPath = config.tsconfigPath ? project.resolvePath(config.tsconfigPath) : undefined return compiler.compileModule(file, fs, diagnostics, tsConfigPath) } function getFunctionParameters(source: Source): string[] { if (!(source instanceof ts.VariableDeclaration || source instanceof ts.FunctionDeclaration)) { return [] } const node = source as ts.VariableDeclaration | ts.VariableDeclaration let functionNode: ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction | undefined if (ts.Node.isFunctionDeclaration(node)) { functionNode = node } else if (ts.Node.isVariableDeclaration(node) && node.getInitializerIfKind(ts.SyntaxKind.FunctionExpression)) { functionNode = node.getInitializerIfKindOrThrow(ts.SyntaxKind.FunctionExpression) } else if (ts.Node.isVariableDeclaration(node) && node.getInitializerIfKind(ts.SyntaxKind.ArrowFunction)) { functionNode = node.getInitializerIfKindOrThrow(ts.SyntaxKind.ArrowFunction) } else { return [] } return functionNode.getParameters().map((param) => param.getName()) ?? [] } async function parseDeclaration( node: ts.FunctionDeclaration | ts.VariableDeclaration, { transform, project, config }: { transform: Transform; config: NowConfig; project: Project } ): Promise> { const result = await transform.toRecord(node.getSourceFile()) if (!result.success) { return result } const record = result.value if (record.getTable() !== 'sys_module') { return { success: false } } const path = record.get('path').asString().getValue() if (NowConfig.legacyPackageResolution(config) || config.scope === 'global') { return { success: true, value: new ModuleFunctionShape(node, { name: node.getName() ?? 'functionModule', path: path, isDefault: node.isDefaultExport(), }), } } const originalPath = (record.getSource() as SourceFileShape).getPath() const relativePath = pathModule.relative(project.getRootDir(), project.resolvePath(originalPath)) return { success: true, value: new ModuleFunctionShape(node, { name: node.getName() ?? 'functionModule', path: path, relativePath: `./${relativePath}`, isDefault: node.isDefaultExport(), }), } } const dependencyIgnoreList = ['@servicenow/glide', '@servicenow/sdk'] function isIgnoredDependency(name: string): boolean { return dependencyIgnoreList.some((dependency) => name.startsWith(dependency)) } function validateAndGetModuleSpecifier(name: string): { name: string; entry?: string } { const scopedRegex = /@[a-z\d][\w\-.]+\/[a-z\d][\w\-.]*/gi scopedRegex.test(name) const idx = scopedRegex.lastIndex const isSubPathImport = name.indexOf('/', idx) if (isSubPathImport > 0) { return { name: name.slice(0, isSubPathImport), entry: name.slice(isSubPathImport + 1) } } return { name } } function isValidRequireCall(callExpression: ts.CallExpression, requirePath: ts.StringLiteral): boolean { const expression = callExpression.getExpression() const isRequire = ts.Node.isIdentifier(expression) && expression.getText() === 'require' const isRelativePath = requirePath.getLiteralText().startsWith('../') || requirePath.getLiteralText().startsWith('./') return isRequire && !isRelativePath } export class ModuleDependencyShape extends Shape { private readonly moduleName: string constructor({ node, moduleName, }: { node: ts.ImportDeclaration | ts.ExportDeclaration | ts.CallExpression moduleName: string }) { super({ source: node }) this.moduleName = moduleName } getModuleName(): string { return this.moduleName } } // TODO: Need to have some invalidation mechanism. Maybe the plugin framework can provide plugins with a managed cache to use for stuff like this? let DEPENDENCY_CACHE: globalThis.Record> = {} // Track external module record IDs for SBOM generation (not persisted to XML) let EXTERNAL_MODULE_IDS: Set = new Set() // Expose cache clearing for tests export function clearDependencyCache() { DEPENDENCY_CACHE = {} EXTERNAL_MODULE_IDS = new Set() } function parseModuleDependency( node: ts.ImportDeclaration | ts.ExportDeclaration | ts.CallExpression ): Result { //Check if this is a type only import and skip it if (node.asKind(ts.SyntaxKind.ImportDeclaration)?.getImportClause()?.isTypeOnly()) { return { success: false } } let moduleName: string | undefined if (ts.Node.isImportDeclaration(node) && !node.isModuleSpecifierRelative()) { moduleName = node.getModuleSpecifierValue() } else if (ts.Node.isExportDeclaration(node) && node.hasModuleSpecifier() && !node.isModuleSpecifierRelative()) { moduleName = node.getModuleSpecifierValue() } else if (ts.Node.isCallExpression(node)) { const args = node.getArguments() const requirePath = args[0] if (ts.Node.isStringLiteral(requirePath) && isValidRequireCall(node, requirePath)) { moduleName = requirePath.getLiteralValue() } } if (!moduleName || isIgnoredDependency(moduleName)) { return { success: false } } const cached = DEPENDENCY_CACHE[moduleName] if (cached && (!cached.success || !cached.value.getOriginalNode().wasForgotten())) { return cached } return (DEPENDENCY_CACHE[moduleName] = { success: true, value: new ModuleDependencyShape({ node, moduleName }), }) } function generateSBOMContent(context: RecordContext) { const { database } = context // Query sys_module records that are tracked as external const moduleRecords = database.query('sys_module').filter((mod) => { const recordId = mod.getId().getValue() return EXTERNAL_MODULE_IDS.has(recordId) }) const sbomBuilder = new SBOMBuilder() for (const mod of moduleRecords) { const modulePath = mod.get('path').asString().getValue() if (modulePath.endsWith('/package.json')) { const moduleContent = mod.get('content').asString().getValue() sbomBuilder.addPackageJson(modulePath, moduleContent) } } return sbomBuilder.generateSBOM(context) } function getModuleDependencyPath( config: NowConfig, module: { name: string; file: string; version: string; packageJson: Package; modulePath?: string } ) { const { name, file, packageJson, modulePath, version } = module if (NowConfig.legacyPackageResolution(config)) { return NowConfig.moduleResolutionPath(config, packageJson, true, name, version, file) } if (modulePath) { return NowConfig.moduleResolutionPath(config, packageJson, true, modulePath, file) } return NowConfig.moduleResolutionPath(config, packageJson, true, NODE_MODULES, name, file) } export const ServerModulePlugin = Plugin.create({ name: 'ServerModulePlugin', files: [ { entryPoint: true, matcher: /[/\\]package\.json$/, }, { entryPoint: true, matcher: (path, { config, project }) => !path.endsWith('.test.ts') && !path.endsWith('.test.js') && project.isInDir(config.serverModulesDir, path), }, ], records: { sys_module: { toFile(record, context) { const { config } = context // If the record is not the sbom, we defer to using the record plugin if (!record.get('path').ifString()?.endsWith('/bom.json')) { return { success: false, } } const sbomContent = generateSBOMContent(context) return { success: true, value: { source: record, name: `sys_module_${record.getId().getValue()}.xml`, category: record.getInstallCategory(), content: ` ${record.getId().getValue()} ${config.scopeId} ${record.get('path').getValue()} ${record.get('external_source').getValue()} `, }, } }, toShape(record) { return { success: true, value: Shape.noOp(record) } }, }, }, shapes: [ { shape: SourceFileShape, fileTypes: ['module'], async toRecord(file, { factory, fs, diagnostics, project, config, packageJson, compiler }) { if (config.type === 'configuration') { throw new Error(`Modules cannot be used in a configuration project`) } const path = file.getPath() if (!path.startsWith(project.resolvePath(config.serverModulesDir))) { return { success: false } } const mappedPath = applyPathMappings(path, config.modulePaths) const resolvedPath = project.resolvePath(mappedPath) const mappedFile = compiler.getSourceFile(resolvedPath) ?? compiler.addSourceFileAtPathIfExists(resolvedPath) if (!mappedFile) { diagnostics.error( file, `Module path was mapped from '${path}' to '${resolvedPath}' but no file exists at the mapped location.` ) return { success: false } } const content = getEmitOutput(mappedFile, { fs, diagnostics, config, project, compiler }) if (!content) { return { success: false } } // Lint local module for Rhino compatibility (no Glide restrictions) if (config.linter.module.enabled && /\.(js|cjs|mjs|ts)$/.test(pathModule.extname(resolvedPath))) { const { LocalModuleLint } = await import('../repack/lint/index.js') const lint = new LocalModuleLint() const lintResult = lint.check(content) if (lintResult) { diagnostics.error(file, `Unsupported APIs detected in module:\n${lintResult}`) } } const relativePath = pathModule.relative(project.getRootDir(), resolvedPath) const sysModulePath = NowConfig.moduleResolutionPath(config, packageJson, false, relativePath) return { success: true, value: await factory.createRecord({ source: file, table: 'sys_module', explicitId: relativePath.replaceAll(/[./\\]/g, '_'), properties: { // Module resolution at runtime requires this format path: sysModulePath, content: Shape.from(file, sanitizeModuleContent(content)) .asString() .withContentType('cdata'), external_source: false, sys_name: sysModulePath, }, }), } }, }, { shape: ModuleDependencyShape, fileTypes: ['module'], // TODO: When managed cache is provided to plugins, cache dependencies that were already handled to avoid reprocessing async toRecord(shape, context) { const { packageJson, diagnostics, fs, logger, project, factory, config } = context if (config.type === 'configuration') { throw new Error(`Modules cannot be used in a configuration project`) } const dependencies = packageJson.dependencies ?? {} const { name: parentName, entry } = validateAndGetModuleSpecifier(shape.getModuleName()) const version = dependencies[parentName] if (!version) { diagnostics.error(shape, `Dependency ${parentName} is not found in package.json`) return { success: false } } const isTrusted = isModuleTrusted(parentName, config.trustedModules, logger) const id = `${parentName}@${version}` const repack = await RepackService.create(logger, fs, project.getRootDir()) const dependencyNodes = await repack.execute({ id, entry: entry ? [entry] : ['.'], legacyPackageResolution: NowConfig.legacyPackageResolution(config), }) if (!dependencyNodes) { throw new Error(`Failed to build dependency ${id}`) } const modules: { id: string; path: string; content: string }[] = [] const { Lint } = await import('../repack/lint/index.js') const lint = new Lint() for (const node of dependencyNodes) { const { packagePath, files, updatedManifest, originalPath } = node const { name, version } = updatedManifest const { modulePath, idPath } = buildDependencyPackagePath( project.getRootDir(), name, version, originalPath ) for (const file of files) { const fileContent = fs.readFileSync(pathModule.join(packagePath, file)).toString('utf-8') if (config.linter.module.enabled && /\.(js|cjs|mjs)$/.test(pathModule.extname(file))) { const result = lint.check(fileContent) if (result) { logger.warn(`Use of unsupported APIs detected in npm dependency ${name}`) logger.warn(result) } } modules.push({ id: NowConfig.legacyPackageResolution(config) ? `${name}@${version}/${file}` : `${idPath}@${version}/${file}`, path: getModuleDependencyPath(config, { name, file, version, packageJson, modulePath, }), content: fileContent, }) } } const records: Record[] = [] for (const m of modules) { const record = await factory.createRecord({ source: shape, table: 'sys_module', explicitId: m.id, properties: { path: m.path, content: Shape.from(shape, sanitizeModuleContent(m.content)) .asString() .withContentType('cdata'), external_source: !isTrusted, sys_name: m.path, }, }) // Track external module IDs for SBOM generation EXTERNAL_MODULE_IDS.add(record.getId().getValue()) records.push(record) } const [first, ...rest] = records if (!first) { return { success: false } } return { success: true, value: first.with(...rest), } }, }, ], nodes: [ { node: 'CallExpression', fileTypes: ['module'], entryPoint: true, toShape: parseModuleDependency }, { node: 'ImportDeclaration', fileTypes: ['module'], entryPoint: true, toShape: parseModuleDependency }, { node: 'ExportDeclaration', fileTypes: ['module'], entryPoint: true, toShape: parseModuleDependency }, { node: 'FunctionDeclaration', fileTypes: ['module'], toShape: parseDeclaration }, { node: 'FunctionExpression', fileTypes: ['module'], toShape(node, context) { const parent = node.getParentIfKind(ts.SyntaxKind.VariableDeclaration) if (!parent) { return { success: false } } return parseDeclaration(parent, context) }, }, { node: 'ArrowFunction', fileTypes: ['module'], toShape(node, context) { const parent = node.getParentIfKind(ts.SyntaxKind.VariableDeclaration) if (!parent) { return { success: false } } return parseDeclaration(parent, context) }, }, ], }) /** * Builds a relative dependency package path from an absolute path. * Handles special cases for monorepo dependencies and pnpm-style paths. * * @param rootDir - The project root directory * @param name - The package name * @param version - The package version * @param path - The absolute path to the package (optional) * @returns Relative path suitable for module resolution */ function buildDependencyPackagePath( rootDir: string, name: string, version: string, path?: string ): { modulePath: string idPath: string } { let relative = pathModule.relative(rootDir, path ?? '') const isMonorepoDependency = relative.includes(`../`) const isPnpm = relative.includes('.pnpm') if (isMonorepoDependency) { // Treat monorepo node_modules as if they are inside app node_modules relative = relative.replaceAll('../', '').replaceAll('+', '/') } if (isPnpm) { // Do our best to turn this pnpm path into an npm path relative = relative.replaceAll('+', '/').replaceAll(`node_modules/.pnpm/${name}@${version}/`, '') } // IDs will resemble the legacy paths and share sys_ids if possible const idPath = relative.replaceAll('node_modules/', '').replaceAll('.pnpm/', '') return { modulePath: relative, idPath, } }