import { path, type Database, type FileSystem, zipSync, type Record, type Factory, type Transform, type Logger, } from '@servicenow/sdk-build-core' import { create } from 'xmlbuilder2' import { CHUNK_SIZE, chunkData, generateId } from './static-content-plugin' import { sha256 } from './now-attach-plugin' import { createNowPackIgnoreFilter, NOW_PACK_IGNORE_FILE } from './now-pack-ignore-utils' /** * Relationship configuration for source artifact tables. * * Defines the hierarchy: M2M → Artifact → Attachment → Attachment Docs */ export const SOURCE_ARTIFACT_RELATIONSHIPS = { sn_glider_source_artifact_m2m: { via: 'application_file', descendant: true, relationships: { sn_glider_source_artifact: { via: 'source_artifact', inverse: true, descendant: true, relationships: { sys_attachment: { via: 'table_sys_id', descendant: true, relationships: { sys_attachment_doc: { via: 'sys_attachment', descendant: true, }, }, }, }, }, }, }, } export async function createArtifactRecord( artifactName: string, parentRecord: Record, factory: Factory ): Promise { return factory.createRecord({ source: parentRecord.getSource(), table: 'sn_glider_source_artifact', explicitId: artifactName, properties: { name: artifactName }, }) } export async function createArtifactM2mRecord( artifactRecord: Record, parentRecord: Record, factory: Factory ): Promise { return factory.createRecord({ source: parentRecord.getSource(), table: 'sn_glider_source_artifact_m2m', explicitId: `${parentRecord.getId().getValue()}-${artifactRecord.getId().getValue()}`, properties: { application_file: parentRecord.getId().getValue(), source_artifact: artifactRecord.getId().getValue(), }, }) } /** * Serializes a metadata record along with its embedded source artifacts to XML. * * Creates a custom XML format where source artifacts, attachments, M2M records, * and the parent metadata record are all embedded within a single XML file. * * The generated XML contains (in order): * 1. The metadata record fields (sys_id, sys_scope, sys_update_name, then metadataProps) * 2. M2M records (sn_glider_source_artifact_m2m) * 3. Artifact records (sn_glider_source_artifact) * 4. Attachment records (sys_attachment) * 5. Attachment doc records (sys_attachment_doc) * 6. delete_multiple cleanup elements * * @param metadataRecord - The main metadata record (e.g., sys_ui_page, sys_aix_experience) * @param metadataProps - List of property names to serialize from the metadata record * @param context - Serialization context (descendants, database, config, transform) */ export async function serializeWithArtifact( metadataRecord: Record, metadataProps: string[], context: { database: Database descendants: { query: (table: string) => Record[] } config: { scope: string; scopeId: string } transform: Transform } ) { const recordUpdate = create().ele('record_update', { table: metadataRecord.getTable() }) const metadataElement = recordUpdate.ele(metadataRecord.getTable(), { action: metadataRecord.getAction(), apply_defaults: 'true', }) metadataElement.ele('sys_id').txt(metadataRecord.getId().getValue()) metadataElement.ele('sys_scope', { display_value: context.config.scope }).txt(context.config.scopeId) const updateName = await context.transform.getUpdateName(metadataRecord) metadataElement.ele('sys_update_name').txt(updateName) for (const prop of metadataProps) { const value = metadataRecord.get(prop) if (value.isDefined()) { const stringValue = value.toString().getValue() const contentType = value.toString().getContentType() if (stringValue) { if (contentType === 'plain') { metadataElement.ele(prop).txt(stringValue) } else { // The ]]> sequence terminates CDATA, so we split it across adjacent CDATA sections if (stringValue.includes(']]>')) { const parts = stringValue.split(']]>') const propElement = metadataElement.ele(prop) propElement.dat(`${parts[0]}]]`) for (let i = 1; i < parts.length; i++) { propElement.dat(`>${parts[i]}`) } } else { metadataElement.ele(prop).dat(stringValue) } } } else { metadataElement.ele(prop) } } } const esLatest = context.descendants .query('sys_es_latest_script') .find((sibling) => sibling.get('id').toString().getValue() === metadataRecord.getId().getValue()) if (esLatest) { const esLatestElement = recordUpdate.ele('sys_es_latest_script', { action: esLatest.getAction(), apply_defaults: 'true', }) esLatestElement.ele('sys_id').txt(esLatest.getId().getValue()) esLatestElement.ele('id').txt(esLatest.get('id').toString().getValue()) esLatestElement.ele('table').txt(esLatest.get('table').toString().getValue()) esLatestElement.ele('use_es_latest').txt(esLatest.get('use_es_latest').toString().getValue()) } context.descendants.query('sn_glider_source_artifact_m2m').forEach((m2m) => { const m2mElement = recordUpdate.ele('sn_glider_source_artifact_m2m', { action: m2m.getAction(), apply_defaults: 'true', }) m2mElement.ele('sys_id').txt(m2m.getId().getValue()) const applicationFile = m2m.get('application_file') if (applicationFile.isDefined()) { const appFileId = applicationFile.isRecord() ? applicationFile.asRecord().getId().getValue() : applicationFile.toString().getValue() m2mElement.ele('application_file').txt(appFileId) } const artifactId = m2m.get('source_artifact') if (artifactId.isDefined()) { const artifactSysId = artifactId.isRecord() ? artifactId.asRecord().getId().getValue() : artifactId.toString().getValue() m2mElement.ele('source_artifact').txt(artifactSysId) } }) context.descendants.query('sn_glider_source_artifact').forEach((artifact) => { const artifactElement = recordUpdate.ele('sn_glider_source_artifact', { action: artifact.getAction(), apply_defaults: 'true', }) artifactElement.ele('sys_id').txt(artifact.getId().getValue()) const artifactName = artifact.get('name') if (artifactName.isDefined()) { artifactElement.ele('name').txt(artifactName.toString().getValue()) } }) context.descendants.query('sys_attachment').forEach((attachment) => { const attachmentElement = recordUpdate.ele('sys_attachment', { action: attachment.getAction(), apply_defaults: 'true', }) attachmentElement.ele('sys_id').txt(attachment.getId().getValue()) attachmentElement.ele('sys_scope', { display_value: context.config.scope }).txt(context.config.scopeId) for (const prop of [ 'table_sys_id', 'table_name', 'file_name', 'content_type', 'size_bytes', 'size_compressed', 'compressed', 'chunk_size_bytes', 'hash', 'average_image_color', 'image_width', 'image_height', ]) { const value = attachment.get(prop) if (value.isDefined()) { attachmentElement.ele(prop).txt(value.toString().getValue()) } } }) context.descendants.query('sys_attachment_doc').forEach((doc) => { const docElement = recordUpdate.ele('sys_attachment_doc', { action: doc.getAction(), apply_defaults: 'true' }) docElement.ele('sys_id').txt(doc.getId().getValue()) const attachmentId = doc.get('sys_attachment') if (attachmentId.isDefined()) { const sysId = attachmentId.isRecord() ? attachmentId.asRecord().getId().getValue() : attachmentId.toString().getValue() docElement.ele('sys_attachment').txt(sysId) } const position = doc.get('position') if (position.isDefined()) { docElement.ele('position').txt(position.toString().getValue()) } const data = doc.get('data') if (data.isDefined()) { docElement.ele('data').txt(data.toString().getValue()) } }) // Emit delete_multiple to clean up stale attachments per artifact. // Attachments use hash-based IDs, so each build may produce a new ID. context.descendants.query('sn_glider_source_artifact').forEach((artifact) => { const artifactId = artifact.getId().getValue() const currentAttachments = context.descendants .query('sys_attachment') .filter((att) => att.get('table_sys_id').toString().getValue() === artifactId) const currentAttachmentIds = currentAttachments.map((att) => att.getId().getValue()) const currentDocIds = context.descendants .query('sys_attachment_doc') .filter((doc) => currentAttachmentIds.includes(doc.get('sys_attachment').toString().getValue())) .map((doc) => doc.getId().getValue()) const attachmentQuery = currentAttachmentIds.length > 0 ? `table_sys_id=${artifactId}^sys_idNOT IN${currentAttachmentIds.join(',')}` : `table_sys_id=${artifactId}` recordUpdate.ele('sys_attachment', { action: 'delete_multiple', query: attachmentQuery }) const docQuery = currentDocIds.length > 0 ? `sys_attachment.table_sys_id=${artifactId}^sys_idNOT IN${currentDocIds.join(',')}` : `sys_attachment.table_sys_id=${artifactId}` recordUpdate.ele('sys_attachment_doc', { action: 'delete_multiple', query: docQuery }) // Query the full database for M2M cleanup (asset M2Ms are descendants of // sys_ux_lib_asset, not of the parent record, so page descendants only has the page M2M). // Skip DELETE-action rows to avoid protecting stale rows on the platform. const currentM2mIds = context.database .query('sn_glider_source_artifact_m2m') .filter((m2m) => { if (m2m.isDeleted()) { return false } const srcArtifact = m2m.get('source_artifact') const srcId = srcArtifact.isRecord() ? srcArtifact.asRecord().getId().getValue() : srcArtifact.toString().getValue() return srcId === artifactId }) .map((m2m) => m2m.getId().getValue()) const m2mQuery = currentM2mIds.length > 0 ? `source_artifact=${artifactId}^sys_idNOT IN${currentM2mIds.join(',')}` : `source_artifact=${artifactId}` recordUpdate.ele('sn_glider_source_artifact_m2m', { action: 'delete_multiple', query: m2mQuery }) }) return { source: metadataRecord, name: `${updateName}.xml`, category: metadataRecord.getInstallCategory(), content: recordUpdate.end({ prettyPrint: true }), } } // sys_update_xml.payload is a MEDIUMTEXT column, so it holds up to 16,777,215 bytes (~16 MB), and // the platform can widen it further (LONGTEXT, up to 4 GB) if ever needed. // The artifact content is stored in sys_attachment_doc.data as base64(zip(raw)) — createAttachmentRecords // below runs the raw files through zipSync once and base64-encodes the result once (there is no // second compression or encoding pass), so: // payload_chars ≈ raw_bytes × (4/3) × Z, where Z is the zip compression ratio (compressed/raw). // Verified against a real production source tree: 28.8 MB raw zipped down to 6.8 MB (Z ≈ 0.24). For // typical TS/JS/JSON app source (Z ≈ 0.25), a full 40 MB raw artifact encodes to ~14 MB of payload // text — about 83% of MEDIUMTEXT's capacity. Content that compresses worse than Z ≈ 0.30 could // overflow MEDIUMTEXT before hitting MAX_TOTAL_SIZE. The limits below are therefore sane guardrails // against runaway artifacts, not a hard platform boundary. // No single file should be able to eat a large fraction of the whole artifact budget, so the // per-file guard is a small slice of the total (1/4) rather than half of it. export const MAX_FILE_SIZE = 10 * 1024 * 1024 // 10 MB — per-file guard (1/4 of total budget) export const MAX_TOTAL_SIZE = 40 * 1024 * 1024 // 40 MB — total artifact budget // Once the running total crosses this fraction of MAX_TOTAL_SIZE, we log a heads-up so users // aren't surprised by a hard failure later (see TOTAL_SIZE_WARNING_RATIO usage below). const TOTAL_SIZE_WARNING_RATIO = 0.85 // How many of the largest files to name when reporting the aggregate size (both the 85% warning // and the over-budget error). The file that triggers either one isn't necessarily the cause — it's // whichever one happened to be read last — so we list the biggest contributors instead. const LARGEST_FILES_TO_REPORT = 5 /** Formats the biggest `LARGEST_FILES_TO_REPORT` entries from `fileSizes` for a log/error message. */ function formatLargestFiles(fileSizes: { filePath: string; size: number }[]): string { return [...fileSizes] .sort((a, b) => b.size - a.size) .slice(0, LARGEST_FILES_TO_REPORT) .map(({ filePath, size }) => ` - ${filePath} (${(size / (1024 * 1024)).toFixed(2)} MB)`) .join('\n') } /** * Reads files from disk and collects them for attachment storage. * * Error handling uses two distinct strategies: * - **Recoverable** (per-file): files that cannot be read or exceed the per-file size limit are * skipped and their paths are collected in the returned `skippedFiles` array. * - **Fatal** (aggregate): if adding a file would push the total over `MAX_TOTAL_SIZE`, an error * is thrown immediately. Since the file that triggers this is just whatever was read last (not * necessarily large itself), the error instead lists the largest files seen so far so users know * what to exclude via `.nowpackignore`. * * If the final total is at or above `TOTAL_SIZE_WARNING_RATIO` of `MAX_TOTAL_SIZE` (but did not * exceed it), a warning is logged once after all files are processed — naming the largest files * across the whole artifact — so users have advance notice before a future addition hits the hard * limit. * * When `useNowPackIgnore` is `true`, a `.nowpackignore` file in `rootDir` is honored like a * `.gitignore` to further filter out files before they are read. Files dropped by the filter are * logged at debug level (an expected, benign exclusion), whereas an entry the filter cannot even * evaluate is recorded in `skippedFiles` like any other recoverable failure. * * @returns Map of file paths to raw buffers, total size in bytes, and array of warning messages for skipped files * @throws {Error} if the cumulative size of valid files would exceed `MAX_TOTAL_SIZE` */ export async function getAttachmentContent( fs: FileSystem, rootDir: string, files: string[], useNowPackIgnore = false, logger?: Logger ): Promise<{ files: Map; totalSize: number; skippedFiles: string[] }> { const fileMap = new Map() let totalSize = 0 const skippedFiles: string[] = [] const fileSizes: { filePath: string; size: number }[] = [] const keepFile = useNowPackIgnore ? createNowPackIgnoreFilter(fs, rootDir) : () => true for (const filePath of files) { let keep: boolean try { keep = keepFile(filePath) } catch (error) { // This does NOT catch a malformed .nowpackignore — that fails hard at filter // construction above (parseNowPackIgnore is deliberately outside this try/catch). The // matcher only throws here on an individual path it cannot evaluate (e.g. one escaping // the project root), so that single file is skipped — mirroring the read-failure // handling below — rather than discarding the entire artifact. skippedFiles.push(`Failed to filter file ${filePath}: ${error}`) continue } if (!keep) { logger?.debug(`Excluding ${filePath} from source artifact (matched ${NOW_PACK_IGNORE_FILE})`) continue } let content: Buffer try { const absolutePath = path.join(rootDir, filePath.replace(/^[/\\]/, '')) content = fs.readFileSync(absolutePath) as Buffer } catch (error) { skippedFiles.push(`Failed to read file ${filePath}: ${error}`) continue } const fileSize = content.length if (fileSize > MAX_FILE_SIZE) { const sizeMB = (fileSize / (1024 * 1024)).toFixed(2) skippedFiles.push(`Skipping file ${filePath}: exceeds max file size (${sizeMB} MB)`) continue } if (totalSize + fileSize > MAX_TOTAL_SIZE) { const currentMB = (totalSize / (1024 * 1024)).toFixed(2) const limitMB = (MAX_TOTAL_SIZE / (1024 * 1024)).toFixed(0) const largestFiles = formatLargestFiles([...fileSizes, { filePath, size: fileSize }]) throw new Error( `Total artifact size would exceed limit (${currentMB}/${limitMB} MB). Largest files:\n${largestFiles}` ) } fileMap.set(filePath, content) fileSizes.push({ filePath, size: fileSize }) totalSize += fileSize } if (totalSize >= MAX_TOTAL_SIZE * TOTAL_SIZE_WARNING_RATIO) { const currentMB = (totalSize / (1024 * 1024)).toFixed(2) const limitMB = (MAX_TOTAL_SIZE / (1024 * 1024)).toFixed(0) const largestFiles = formatLargestFiles(fileSizes) logger?.warn( `Source artifact size (${currentMB}/${limitMB} MB) is approaching the total budget. Largest files:\n${largestFiles}` ) } return { files: fileMap, totalSize, skippedFiles } } /** * Creates attachment and attachment_doc records for the artifact. * * Attachments are compressed using zip and chunked for storage. The attachment ID includes * a hash prefix to ensure immutability (each content version gets a unique ID). */ export async function createAttachmentRecords( factory: Factory, artifactRecord: Record, files: Map, totalSize: number ): Promise { // Build zip entries with a fixed mtime for deterministic output. // ZIP format only supports dates 1980-2099; we use the minimum valid date. // new Date(1982, 0, 1) creates midnight January 1, 1982 in local time, // which fflate's local-time date encoding renders identically in any timezone. const FIXED_MTIME = new Date(1982, 0, 1) const zipEntries: { [path: string]: [Buffer, { mtime: Date }] } = {} for (const [filePath, content] of files) { zipEntries[filePath] = [content, { mtime: FIXED_MTIME }] } const zipped = zipSync(zipEntries) const compressedData = Buffer.from(zipped) const hash = await sha256(compressedData) // Uniform chunking — no special header split const allChunks = chunkData(compressedData.toString('base64')) // Deterministic sys_id derived from artifact + content hash — no keys registry entry needed. // Same content always produces the same attachment ID without inflating keys.json. const attachmentSysId = generateId(artifactRecord.getId().getValue(), 'sys_attachment', hash) const attachment = await factory.createRecord({ source: artifactRecord.getSource(), table: 'sys_attachment', properties: { sys_id: attachmentSysId, average_image_color: '', chunk_size_bytes: CHUNK_SIZE, compressed: true, content_type: 'application/zip', hash, image_height: '', image_width: '', size_bytes: totalSize, size_compressed: compressedData.length, file_name: `${artifactRecord.get('name').getValue()}.zip`, table_name: artifactRecord.getTable(), table_sys_id: artifactRecord.getId().getValue(), }, }) const attachmentDocs: Record[] = [] for (let i = 0; i < allChunks.length; i++) { const doc = await factory.createRecord({ source: artifactRecord.getSource(), table: 'sys_attachment_doc', properties: { sys_id: generateId(attachmentSysId, 'sys_attachment_doc', i), data: allChunks[i], position: i, sys_attachment: attachment.getId().getValue(), }, }) attachmentDocs.push(doc) } return [attachment, ...attachmentDocs] }