import MD5 from 'md5.js' import * as mime from 'mime-types' import picomatch from 'picomatch' import { Shape, Plugin, type Record, type NowConfig, unloadBuilder, gzipSync } from '@servicenow/sdk-build-core' import { path as pathModule } from '@servicenow/sdk-build-core' import { INVALID_XML_CHARACTERS } from './utils' import { resolveConfigValue, type ResolverContext } from './config-resolver' import { TRANSLATIONS_SUFFIX } from '@servicenow/isomorphic-rollup' // based on tectonic code for attachments // https://code.devsnc.com/dev/sn-tectonic/blob/b3ab42ce742158cb5a0d00efd540b97eeafcbdd7/core/metadata-transform-san-diego/utils/index.js#L66 const MAX_INLINE_CONTENT_SIZE = 4194304 // https://code.devsnc.com/dev/sn-tectonic/blob/b3ab42ce742158cb5a0d00efd540b97eeafcbdd7/core/metadata-transform-san-diego/attachment/index.js#L7 export const CHUNK_SIZE = 933336 const base64 = (data: Uint8Array | Buffer | string): string => Buffer.from(data).toString('base64') export const chunkData = (data: string): string[] => { const numChunks = Math.ceil(data.length / CHUNK_SIZE) const chunks: string[] = new Array(numChunks) for (let chunk = 0, offset = 0; chunk < numChunks; chunk += 1, offset += CHUNK_SIZE) { chunks[chunk] = data.substring(offset, offset + CHUNK_SIZE) } return chunks } export const generateId = (...parts: Array): string => new MD5().update(parts.join(':')).digest('hex') // based on tectonic code for validating XML content // https://code.devsnc.com/dev/sn-tectonic/blob/b3ab42ce742158cb5a0d00efd540b97eeafcbdd7/core/metadata-transform-san-diego/utils/index.js#L51-L63 function validateCdata(data: string): boolean { if (data && data.includes(' { const recordBuilder = unloadBuilder() for (const record of mainRecord.flat()) { const builder = recordBuilder.record(record) record .entries() .sort(([a], [b]) => a.localeCompare(b)) // Sort keys to make outputs more deterministic .forEach(([prop, shape]) => builder.field(prop, shape)) } return { success: true, value: { source: mainRecord, name: `${mainRecord.getTable()}_${mainRecord.getId().getValue()}.xml`, category: mainRecord.getInstallCategory(), content: recordBuilder.end(), }, } } const attachmentRelationships = { sys_attachment: { via: 'table_sys_id', descendant: true, relationships: { sys_attachment_doc: { via: 'sys_attachment', descendant: true, }, }, }, } const sourceArtifactRelationships = { sn_glider_source_artifact_m2m: { via: 'application_file', descendant: true, relationships: { sn_glider_source_artifact: { via: 'source_artifact', inverse: true, descendant: true, }, }, }, } const toNoOpShape = (record: Record) => { return { success: true, value: Shape.noOp(record) } } /** * A resolved static content mapping entry. * * @property pattern - The glob pattern (after pseudo-property resolution) used to match files. * @property publicPath - The URL path prefix applied when naming the published asset. * @property base - The non-glob prefix of `pattern`, used to compute relative file paths. * @property isMatch - A compiled picomatch matcher for the pattern. */ type StaticContentEntry = { pattern: string publicPath: string base: string isMatch: picomatch.Matcher } /** Cache of built entries keyed on the NowConfig object identity to avoid re-computation. */ const staticContentEntriesCache = new WeakMap() /** * Returns the resolved list of {@link StaticContentEntry} items for the given config, * building and caching them on first access. */ function getStaticContentEntries(config: NowConfig, resolver: ResolverContext): StaticContentEntry[] { let entries = staticContentEntriesCache.get(config) if (!entries) { entries = buildStaticContentEntries(config, resolver) staticContentEntriesCache.set(config, entries) } return entries } /** * Builds the ordered list of static content entries from `staticContent.assets`. * * The default `staticContent.buildDir` entry is expected to be provided by the * project layer — this function only processes entries already present in config. * Pseudo-property tokens (e.g. `[$config.scope]`, `[$config.aiux.basename]`) in * both patterns and `publicPath` values are resolved via {@link resolveConfigValue}. */ function buildStaticContentEntries(config: NowConfig, resolver: ResolverContext): StaticContentEntry[] { if (!config.staticContent.assets) { return [] } return Object.entries(config.staticContent.assets).map(([rawPattern, options]) => { const pattern = resolveConfigValue(rawPattern, resolver) const publicPath = resolveConfigValue(options.publicPath, resolver) const scan = picomatch.scan(pattern) return { pattern, publicPath, base: scan.base, isMatch: picomatch(pattern, { dot: true }), } }) } /** * Finds the first {@link StaticContentEntry} whose glob pattern matches the * given absolute file path (resolved relative to `rootDir`). */ function findMatchingEntry( entries: StaticContentEntry[], absolutePath: string, rootDir: string ): StaticContentEntry | undefined { const relativePath = pathModule.relative(rootDir, absolutePath) return entries.find((entry) => entry.isMatch(relativePath)) } export const StaticContentPlugin = Plugin.create({ name: 'StaticContentPlugin', noTelemetry: true, records: { // TODO: remove when we have generic attachment support sys_attachment: { toShape: toNoOpShape, }, // TODO: remove when we have generic attachment support sys_attachment_doc: { toShape: toNoOpShape, }, sys_ui_message: { toShape: toNoOpShape, }, sys_ux_lib_asset: { coalesce: ['name'], relationships: { ...attachmentRelationships, ...sourceArtifactRelationships }, toShape: toNoOpShape, toFile: async (mainRecord, context) => { const existingRelated = mainRecord.flat().slice(1) const m2mRecords = context.descendants.query('sn_glider_source_artifact_m2m') return multipleRecordsToFile(mainRecord.with(...existingRelated, ...m2mRecords)) }, }, db_image: { relationships: attachmentRelationships, toShape: toNoOpShape, toFile: multipleRecordsToFile, }, sys_ux_theme_asset: { relationships: attachmentRelationships, toShape: toNoOpShape, toFile: multipleRecordsToFile, }, }, files: [ { entryPoint: true, matcher: (path, { config, project, fs }) => { const resolver = { config, rootDir: project.getRootDir(), fs } const entries = getStaticContentEntries(config, resolver) const rel = pathModule.relative(project.getRootDir(), path) const matches = entries.some((entry) => entry.isMatch(rel)) return matches }, async toRecord(file, { project, config, factory, logger, fs }) { const { path } = file const resolver = { config, rootDir: project.getRootDir(), fs } const entries = getStaticContentEntries(config, resolver) const matchedEntry = findMatchingEntry(entries, path, project.getRootDir()) if (!matchedEntry) { return { success: false } } const relativePath = pathModule.relative(project.resolvePath(matchedEntry.base), path) const ext = pathModule.extname(relativePath) const normalizedExt = ext.replace('dbx', '') const mimeType = mime.lookup(normalizedExt) // TODO: support binary file content so we don't have to read the contents again here const rawFileContent = fs.readFileSync(path) const stringFileContent = file.content const hash = new MD5().update(rawFileContent).digest('hex') let tableName: string | undefined let assetName: string | undefined const uxAssetProperties = { sys_package: config.scopeId, mime_type: mimeType, checksum: hash, } const recordProperties = {} let useAttachment = false const logSkippedWithReason = (reason: string) => { // it would be nice to use diagnostics.warn here, but we don't have a compatible source for this file // so we use logger instead, similar to what the server-module-plugin does // https://code.devsnc.com/dev/fluent/blob/fbb5dfcf19dd3bf1ad8700de939a12817b16d0ad/packages/build-plugins/src/server-module-plugin.ts#L367-L368 logger.warn(`Skipping packaging of '${relativePath}'. ${reason}`) } if (rawFileContent.length > MAX_INLINE_CONTENT_SIZE || !validateCdata(stringFileContent)) { useAttachment = true } Object.assign(uxAssetProperties, { is_attachment: useAttachment, content: useAttachment ? undefined : Shape.from(file, stringFileContent).asString().withContentType('cdata'), }) if (mimeType === 'text/html') { // This content will be handled by the UiPage referencing it } else if (relativePath.endsWith('.ui-source-manifest.json')) { // Build-time manifest produced by the uiPageSourceManifest rollup plugin. // Consumed by UiPage during build to determine which source files to include // in the source artifact record. Not a deployable asset. } else if (relativePath.endsWith(TRANSLATIONS_SUFFIX)) { // Build-time asset produced by collectSnTranslate() rollup plugin. // Create sys_ui_message records for each collected translation message key. // Messages can be plain strings (t('key')) or objects (t({ code, message, ... })). const parsed = JSON.parse(stringFileContent) as { messages: Array } const { messages } = parsed if (messages.length > 0) { const messageRecords = await Promise.all( messages.map((entry) => { const key = typeof entry === 'string' ? entry : (entry['code'] ?? '') const messageText = typeof entry === 'string' ? entry : (entry['message'] ?? key) return factory.createRecord({ source: file, explicitId: key, table: 'sys_ui_message', properties: { code: config.scope, key, language: config.defaultLanguage, message: messageText, }, }) }) ) const [first, ...rest] = messageRecords if (!first) { return { success: false } } return { success: true, value: first.with(...rest) } } return { success: false } } else if (mimeType === 'application/javascript') { tableName = 'sys_ux_lib_asset' assetName = pathModule.join( matchedEntry.publicPath, relativePath.substring(0, relativePath.length - ext.length) ) Object.assign(recordProperties, { ...uxAssetProperties, name: assetName, category: 'component', es_module: true, engine: 21, }) } else if (normalizedExt === '.map') { tableName = 'sys_ux_lib_asset' assetName = pathModule.join(matchedEntry.publicPath, relativePath.replace('dbx', '')) Object.assign(recordProperties, { ...uxAssetProperties, name: assetName, category: 'source_map', es_module: false, }) } else if (!mimeType) { logSkippedWithReason('Unknown content type.') } else if (mimeType.indexOf('image/') === 0) { useAttachment = true tableName = 'db_image' assetName = pathModule.join(matchedEntry.publicPath, relativePath) Object.assign(recordProperties, { name: assetName, size_bytes: rawFileContent.length, active: true, }) } else if (mimeType.indexOf('font/') === 0 || mimeType === 'text/css') { useAttachment = true tableName = 'sys_ux_theme_asset' assetName = relativePath Object.assign(recordProperties, { name: assetName, // this is a required field, and the only other value allowed is 'image' // it doesn't affect the content type of the response category: 'font', }) } else { logSkippedWithReason(`Unsupported content type ${mimeType}.`) } if (tableName && assetName) { const mainRecord = await factory.createRecord({ source: file, table: tableName, explicitId: assetName, properties: recordProperties, }) const mainRecordId = mainRecord.getId().getValue() const overrideMainRecordProperties = {} const attachmentRecords: Record[] = [] if (useAttachment) { const attachmentRecordId = generateId(mainRecordId, 'sys_attachment', assetName) const zipEncodedContent = gzipSync(rawFileContent, { mtime: 0 }) const attachmentRecord = await factory.createRecord({ source: file, table: 'sys_attachment', properties: { sys_id: attachmentRecordId, file_name: assetName, hash: hash, table_name: tableName, table_sys_id: mainRecordId, size_bytes: rawFileContent.length, size_compressed: zipEncodedContent.length, chunk_size_bytes: CHUNK_SIZE, compressed: true, content_type: mimeType, state: 'available', }, }) attachmentRecords.push(attachmentRecord) if (tableName === 'db_image') { Object.assign(overrideMainRecordProperties, { image: attachmentRecordId }) } const base64Encoding = base64(zipEncodedContent) const chunks = chunkData(base64Encoding) for (let i = 0; i < chunks.length; i++) { const sysId = generateId(mainRecordId, 'sys_attachment_doc', assetName, i) attachmentRecords.push( await factory.createRecord({ source: file, table: 'sys_attachment_doc', properties: { sys_id: sysId, data: chunks[i], position: i, length: chunks[i]!.length, sys_attachment: attachmentRecordId, }, }) ) } } const mergedRecord = mainRecord.merge(overrideMainRecordProperties) return { success: true, value: mergedRecord.with(...attachmentRecords), } } return { success: false } }, }, ], })