import { CallExpressionShape, type Database, deleteMultipleDiff, type Diagnostics, IdentifierShape, ObjectShape, Plugin, PropertyAccessShape, type Record as RecordShape, type Shape, type Source, type StringShape, TemplateExpressionShape, TemplateSpanShape, type TemplateSubstitutions, ts, UndefinedShape, VariableStatementShape, } from '@servicenow/sdk-build-core' import { NowIdShape } from '../now-id-plugin' import { ArrowFunctionShape } from '../arrow-function-plugin' import { create } from 'xmlbuilder2' import { Test } from '@servicenow/sdk-core/runtime/app' import { type ATFVariableInfo, type Category, ToShapeStepConfigs, ToRecordStepConfigs, type StepMetadata, } from './step-configs' import { durationFieldToXML, formatDateToPlatformFormat, parseGlideDuration } from '@servicenow/sdk-build-core' import type { Duration } from '@servicenow/sdk/core' export const TestPlugin = Plugin.create({ name: 'TestPlugin', records: { sys_atf_test: { relationships: { sys_atf_step: { via: 'test', descendant: true, relationships: { sys_variable_value: { descendant: true, via: 'document_key', }, sys_element_mapping: { descendant: true, via: 'id', }, }, }, }, toShape(record, { descendants, database }) { const arrowParameter = getArrowFunctionParameter(record) const elementMappingRecords = descendants.query('sys_element_mapping') const stepsWhichShouldDeclareAVariable = getStepIdsThatAreBeingReferencedByOtherSteps(elementMappingRecords) const elementMappingValueMap = getFieldValuesFromElementMappingRecords(elementMappingRecords) const stepRecords = descendants .query('sys_atf_step') .sort((a, b) => a.get('order').toNumber().getValue() - b.get('order').toNumber().getValue()) if (anyStepsAreUnknown(stepRecords, record.get('name').toString().getValue())) { return { success: false } } const stepVarNames = getVariableNamesForSteps(stepRecords) const statements = stepRecords.map((step) => { const stepConfigId = step.get('step_config').toString().getValue() const stepId = step.getId().getValue() const stepConfig = ToShapeStepConfigs[stepConfigId]! const gemIdentifier = stepsWhichShouldDeclareAVariable.has(stepId) ? stepVarNames.get(stepId) : undefined const variables = descendants .query('sys_variable_value', { document: 'sys_atf_step', document_key: stepId }) .map((svvRecord) => { const variableShape = svvRecord.get('variable') const variableInfo = stepConfig.variables[variableShape.getValue() as string] if (!variableInfo) { throw new Error( `sys_variable_value variable ${variableShape.getValue()} does not correspond to any known ATF input variables for step type '${stepConfig.name}'` ) } const rawValue = elementMappingValueMap.get(`${stepId}:${variableInfo.field}`) ?? svvRecord.get('value') const deserializedValue = serializeValueToShape( rawValue?.ifDefined()?.toString().getValue(), variableInfo.type, variableInfo.mandatory ) if (deserializedValue === undefined) { return undefined } let value = getElementMappingValues(svvRecord, deserializedValue, stepVarNames) if (typeof value === 'string' && value.startsWith('REL:')) { value = resolveRelationshipReference(value, database, svvRecord.getSource()) } if (variableInfo.default !== undefined && value === variableInfo.default) { // no need to set the value if its already default return undefined } const name = variableInfo.name return { name, value, } }) .filter((v) => v !== undefined) .map((v) => ({ [v.name]: v.value })) .reduce((acc, v) => ({ ...acc, ...v }), {}) const rawTimeoutValue = step.get('timeout').ifString()?.getValue() const timeout = parseGlideDuration(rawTimeoutValue) const callExpression = new CallExpressionShape({ source: step, callee: `${arrowParameter}.${stepConfig?.callName}`, args: [ new ObjectShape({ source: step, properties: step.transform(({ $, merge }) => ({ $id: $.val(NowIdShape.from(step)), active: $.toBoolean().def(true), description: $.def(''), notes: $.def(''), warning: $.from('warning_message').def(''), timeout: $.val(timeout), [merge]: $.val(variables).def({}), })), }), ], }) return gemIdentifier ? new VariableStatementShape({ source: step, variableName: gemIdentifier, initializer: callExpression, }) : callExpression }) const arrowFunction = new ArrowFunctionShape({ source: record, parameters: [arrowParameter], statements, }) return { success: true, value: new CallExpressionShape({ source: record, callee: Test.name, args: [ record.transform(({ $ }) => ({ $id: $.val(NowIdShape.from(record)), name: $, description: $.def(''), active: $.toBoolean().def(true), failOnServerError: $.from('fail_on_server_error').toBoolean().def(false), })), arrowFunction, ], }), } }, async diff(existing, incoming, descendants, context) { return deleteMultipleDiff(existing, incoming, descendants, context) }, }, sys_variable_value: { coalesce: ['document', 'document_key', 'variable'], }, sys_atf_step: { composite: true, toFile(step, { database, config }) { const recordUpdate = create().ele('record_update', { table: 'sys_atf_step' }) const stepElement = recordUpdate.ele('sys_atf_step', { action: step.getAction(), apply_defaults: 'true', }) const stepId = step.getId().getValue() if (!stepId) { throw new Error('Invalid step id') } stepElement.ele('sys_id').txt(stepId) stepElement.ele('sys_scope', { display_value: config.scope }).txt(config.scopeId) Object.entries(step.properties()).forEach(([key, value]) => { stepElement.ele(key).txt(value.ifDefined()?.toString()?.getValue() ?? '') }) const variableValues = database.query('sys_variable_value', { document: 'sys_atf_step', document_key: step.getId(), }) recordUpdate.ele('sys_variable_value', { action: 'delete_multiple', query: `document_key=${stepId}`, }) variableValues.forEach((varRecord) => { const variableElement = recordUpdate.ele('sys_variable_value', { action: step.getAction(), apply_defaults: 'true', }) variableElement.ele('sys_id').txt(varRecord.getId().getValue()) Object.entries(varRecord.properties()).forEach(([key, value]) => { variableElement.ele(key).txt(value instanceof UndefinedShape ? '' : value.toString().getValue()) }) }) const elementMappingRecords = database.query('sys_element_mapping', { id: stepId }) if (elementMappingRecords.length > 0) { recordUpdate.ele('sys_element_mapping', { action: 'delete_multiple', query: `id=${stepId}`, }) } elementMappingRecords.forEach((record) => { const elementMappingElement = recordUpdate.ele('sys_element_mapping', { action: step.getAction(), apply_defaults: 'true', }) elementMappingElement.ele('sys_id').txt(record.getId().getValue()) Object.entries(record.properties()).forEach(([key, value]) => { elementMappingElement.ele(key).txt(value.toString().getValue()) }) }) return { success: true, value: { source: step, name: `sys_atf_step_${step.getId().getValue()}.xml`, category: step.getInstallCategory(), content: recordUpdate.end({ prettyPrint: true }), }, } }, }, sys_element_mapping: { coalesce: ['field', 'table', 'id'], }, }, shapes: [ { shape: CallExpressionShape, fileTypes: ['fluent'], async toRecord(callExpression, { factory, diagnostics }) { if (callExpression.getCallee() !== Test.name) { return { success: false } } const prop = callExpression.getArgument(0).asObject() const arrowFunc = callExpression.getArgument(1).if(ArrowFunctionShape) const testId = prop.get('$id') const steps = arrowFunc ? getStepInfoFromArrowFunction(arrowFunc, diagnostics) : [] if (steps.some((step) => !step)) { return { success: false } } const testRecord = await factory.createRecord({ source: callExpression, table: 'sys_atf_test', explicitId: testId, properties: prop.transform(({ $ }) => ({ name: $, active: $.def(true), description: $.def(''), fail_on_server_error: $.from('failOnServerError').def(false), })), }) const stepRecords = await Promise.all( steps.map(async (step, index) => { const rawTimeout = step.values.get('timeout') const timeout = rawTimeout.isUndefined() ? step.defaultTimeout : convertTimeoutToSystemFormat(rawTimeout) return await factory.createRecord({ table: 'sys_atf_step', source: step.source, explicitId: step.values.get('$id'), properties: step.values.transform(({ $ }) => ({ active: $.toBoolean().def(true), description: $, display_name: $.val(step.displayName), notes: $, order: $.val(index + 1), step_config: $.val(step.stepConfigId), table: $.val(step.table), test: $.val(testRecord.getId()), timeout: $.val(timeout), warning_message: $.from('warning'), })), }) }) ) const stepAndVariableValueRecords: RecordShape[] = [] for (const [index, step] of steps.entries()) { for (const [varName, atfVariableInfo] of Object.entries(step.variables)) { const valueShape = step.values.get(varName, false) let elementMappingRecord: RecordShape | undefined const value = createGemExpressionsFromShape(valueShape, stepRecords, diagnostics) if (value !== undefined) { elementMappingRecord = await factory.createRecord({ table: 'sys_element_mapping', source: step.source, properties: { field: atfVariableInfo.field, id: stepRecords[index]?.getId(), table: `var__m_atf_input_variable_${step.stepConfigId}`, value: serializeValueToRecord(value, atfVariableInfo.type, atfVariableInfo.default), }, }) } const resolvableRecord = valueShape.ifResolvable()?.resolve().ifRecord() stepAndVariableValueRecords.push( await factory.createRecord({ table: 'sys_variable_value', source: step.source, properties: { document: 'sys_atf_step', document_key: stepRecords[index]?.getId(), value: serializeValueToRecord( value ?? valueShape .ifArray() ?.getElements() .map((v) => { const record = v.ifRecord() return record ? recordReferenceValue(record) : v.getValue() }) ?? (resolvableRecord ? recordReferenceValue(resolvableRecord) : undefined) ?? valueShape.getValue(), atfVariableInfo.type, atfVariableInfo.default ), variable: atfVariableInfo.inputVariableId, order: atfVariableInfo.order, }, }) ) if (elementMappingRecord) { stepAndVariableValueRecords.push(elementMappingRecord) } } } return { success: true, value: testRecord.with(...stepRecords, ...stepAndVariableValueRecords), } }, }, ], }) const UNSUPPORTED_STEP_CONFIGS = { '38907e937322130007d738682bf6a742': 'Component State Validation (Custom UI)', '475e0de3d732130089fca2285e610361': 'Assert Text on Page (Custom UI)', b4758c7453370300c792ddeeff7b128d: 'Component Value Validation (Custom UI)', def25c4b73730300c79260bdfaf6a700: 'Click Component (Custom UI)', e5dd168473330300c79260bdfaf6a794: 'Set Component Values (Custom UI)', fc7e65d577332300e46abe41a9106106: 'Open Service Portal Page', } function getFieldValuesFromElementMappingRecords(elementMappingRecords: RecordShape[]) { return elementMappingRecords .filter((em) => em.get('value').getValue()) .reduce((map, em) => { const field = em.get('field').getValue() const stepId = em.get('id').getValue() return map.set(`${stepId}:${field}`, em.get('value')) }, new Map()) } function convertTimeoutToSystemFormat(timeoutShape: Shape) { const timeoutDuration = timeoutShape.ifObject()?.getValue() as Duration | undefined const timeoutDate = timeoutDuration && durationFieldToXML(timeoutDuration) return timeoutDate && formatDateToPlatformFormat(timeoutDate) } function anyStepsAreUnknown(stepRecords: RecordShape[], testName: string) { return stepRecords.some((stepRecord) => { const stepConfigId = stepRecord.get('step_config').toString().getValue() if (!ToShapeStepConfigs[stepConfigId as keyof typeof ToShapeStepConfigs]) { const stepConfigName = UNSUPPORTED_STEP_CONFIGS[stepConfigId as keyof typeof UNSUPPORTED_STEP_CONFIGS] ?? stepConfigId console.error( `Unable to transform ATF test '${testName}' because of unsupported step_config: '${stepConfigName}'` ) return true } return false }) } function getArrowFunctionParameter(record: RecordShape) { const source = record.getOriginalSource() return ( (ts.Node.isNode(source) && source.isKind(ts.SyntaxKind.CallExpression) && source.getArguments()[1]?.asKind(ts.SyntaxKind.ArrowFunction)?.getParameters()[0]?.getName()) || 'atf' ) } function propertyAccessToGemExpression( propertyAccess: PropertyAccessShape, stepRecords: RecordShape[], diagnostics: Diagnostics ) { const split = propertyAccess.getCode().split('.') if (!split[0]) { diagnostics.error(propertyAccess, `Invalid expression`) return undefined } const sourceStep = stepRecords.find( (rec) => rec.getSource() === propertyAccess.getFirstElement().resolve().if(CallExpressionShape) ) if (!sourceStep) { diagnostics.error(propertyAccess, `Invalid expression, expecting a reference to an ATF step`) } const stepId = sourceStep?.getId().getValue() const elementPath = split.slice(1).join('.') const suffix = elementPath ? `.${elementPath}` : '' return `{{step['${stepId}']${suffix}}}` } function createGemExpressionsFromShape( shape: Shape, stepRecords: RecordShape[], diagnostics: Diagnostics ): string | Record | undefined { if (shape instanceof PropertyAccessShape) { return propertyAccessToGemExpression(shape, stepRecords, diagnostics) } if (shape instanceof TemplateExpressionShape) { const gemExpressions = shape .getSpans() .map((span) => propertyAccessToGemExpression(span.getExpression().as(PropertyAccessShape), stepRecords, diagnostics) ) .reduce((acc, gemExpression, index) => ({ ...acc, [index]: gemExpression }), {}) as TemplateSubstitutions return shape.getValue(gemExpressions) } if (shape instanceof ObjectShape) { const obj: Record = {} let gemExpressionFound = false for (const key in shape.properties()) { const value = createGemExpressionsFromShape(shape.get(key, false), stepRecords, diagnostics) gemExpressionFound = gemExpressionFound || value !== undefined obj[key] = value || shape.get(key).getValue() } return gemExpressionFound ? obj : undefined } return undefined } function getStepInfoFromArrowFunction(arrowFunc: ArrowFunctionShape, diagnostics: Diagnostics) { return arrowFunc .getStatements() .filter((statement) => statement instanceof CallExpressionShape || statement instanceof VariableStatementShape) .map((statement) => getStepInfoFromAtfCallExpression(statement, diagnostics)) .filter((step) => step) as StepInfo[] } type StepInfo = { displayName: string source: Source stepConfigId: string table: StringShape | undefined variables: Record values: ObjectShape identifier: string | undefined defaultTimeout?: string } function getStepInfoFromAtfCallExpression( statement: CallExpressionShape | VariableStatementShape, diagnostics: Diagnostics ): StepInfo | undefined { let callExpression: CallExpressionShape let identifier: string | undefined if (statement instanceof VariableStatementShape) { callExpression = statement.getInitializer() as CallExpressionShape identifier = statement.getVariableName().getName() } else if (statement instanceof CallExpressionShape) { callExpression = statement } else { return undefined } const [, category, funcName] = callExpression.getCallee().split('.') as [never, Category?, string?] if (!category || !funcName) { diagnostics.info(callExpression, 'Invalid ATF step') return undefined } const categoryKey = category if (!ToRecordStepConfigs[categoryKey]) { diagnostics.warn(callExpression, `Unknown category: ${category}`) return undefined } const categorySteps = ToRecordStepConfigs[categoryKey] if (!(funcName in categorySteps)) { diagnostics.warn(callExpression, `Unknown step: ${funcName}`) return undefined } const stepConfig = categorySteps[funcName as keyof typeof categorySteps] as StepMetadata const values = callExpression.getArgument(0).asObject() return { displayName: stepConfig.name, stepConfigId: stepConfig.stepConfigId, variables: stepConfig.variables, table: values.get('table')?.ifString(), source: callExpression, identifier, defaultTimeout: stepConfig.defaultTimeout as string, values, } } function getVariableNamesForSteps(stepRecords: RecordShape[]) { return stepRecords.reduce((map, step, index) => { const source = step.getOriginalSource() const name = (ts.Node.isNode(source) && source.getParentIfKind(ts.SyntaxKind.VariableDeclaration)?.getName()) || `step${index + 1}` return map.set(step.getId().getValue(), name) }, new Map()) } const GemRegex = /{{step\[(?:'|.{6})(?[0-9a-f]{32})(?:'|.{6})\]\.(?.*?)}}/gm function getStepIdsThatAreBeingReferencedByOtherSteps(elementMappingRecord: RecordShape[]) { return elementMappingRecord .flatMap((rec) => { const str = rec.get('value').toString().getValue() const matches = str.matchAll(GemRegex) return Array.from(matches).map((m) => { const [, stepId] = m return stepId }) }) .filter((stepId) => stepId !== undefined) .reduce((acc, stepId) => acc.add(stepId), new Set()) } function variableElementPathToPropertyAccess(source: Source, propertyAccessPath: string) { const elements = propertyAccessPath.split('.').map((e) => new IdentifierShape({ source, name: e })) return new PropertyAccessShape({ source, elements: [elements[0], elements[1], ...elements.slice(2)], }) } function getElementMappingValues( source: Source, value: unknown, stepIdToVarName: Map ): object | object[] | string | undefined { if (typeof value === 'object' && value !== null) { if (Array.isArray(value)) { return value.map((v) => getElementMappingValues(source, v, stepIdToVarName)) } else { const obj: Record = {} const valueObj = value for (const key in valueObj) { obj[key] = getElementMappingValues(source, valueObj[key as keyof typeof valueObj], stepIdToVarName) } return obj } } const potentialGemExpression = typeof value === 'string' ? value : undefined if (!potentialGemExpression) { return value as string } const [matched] = Array.from(potentialGemExpression.matchAll(GemRegex)) const simpleGemExpression = matched && matched[0] === potentialGemExpression if (!matched) { return value as string } // If the GEM expression is simple, it only contains the GEM // expression and nothing else. This means we can transform it // a PropertyAccessExpression, otherwise we'll need to create // a TemplateExpression if (simpleGemExpression) { const stepId = matched.groups!['stepId']! const elementPath = matched.groups?.['elementPath'] const varName = stepIdToVarName.get(stepId) if (!varName) { return undefined } const lastIdentifier = elementPath?.split('.')[0] if (!lastIdentifier) { throw new Error(`Invalid GEM expression found in: '${potentialGemExpression}'`) } return variableElementPathToPropertyAccess(source, `${varName}.${elementPath}`) } const matches = Array.from(potentialGemExpression.matchAll(GemRegex)) if (!matches[0]) { return undefined } const spans: TemplateSpanShape[] = [] const head = potentialGemExpression.substring(0, matches[0].index) for (let i = 0; i < matches.length; i++) { const m = matches[i]! const gem = m[0] const nextGemIndex = matches[i + 1]?.index const tail = potentialGemExpression.substring(m.index + gem.length, nextGemIndex) const stepId = m.groups?.['stepId'] as string const elementPath = m.groups?.['elementPath'] ?? '' const varName = stepIdToVarName.get(stepId) spans.push( new TemplateSpanShape({ source, literalText: tail, expression: variableElementPathToPropertyAccess(source, `${varName}.${elementPath}`), }) ) } return new TemplateExpressionShape({ source, literalText: head, spans: spans, }) } function recordReferenceValue(record: RecordShape): string { const id = record.getId().getValue() return record.getTable() === 'sys_relationship' ? `REL:${id}` : id } function resolveRelationshipReference(value: string, database: Database, source: Source): Shape | string { const guid = value.slice(4) const relationship = database.get('sys_relationship', guid) if (!relationship) { return value } const originalSource = relationship.getOriginalSource() if (!ts.Node.isNode(originalSource)) { return value } const variableDeclaration = originalSource.getParentIfKind(ts.SyntaxKind.VariableDeclaration) if (!variableDeclaration) { return value } return new IdentifierShape({ source, name: variableDeclaration.getName() }) } function objectToEncodedQuery(objectShape: Record) { return Object.entries(objectShape) .map(([key, value]) => { const escapedValue = value.toString().replaceAll('^', '^^') return `${key}=${escapedValue}` }) .concat('EQ') .join('^') } export function serializeValueToRecord(value: unknown, type: string, defaultValue: unknown) { value = value === undefined ? defaultValue : value switch (type) { case 'field_list': case 'glide_list': case 'slushbucket': return Array.isArray(value) ? value.join(',') : value case 'string': { return value ? value : undefined } case 'simple_name_values': { if (typeof value !== 'object' || value === null || Object.keys(value).length === 0) { return '' } return JSON.stringify(value) } case 'template_value': return value && objectToEncodedQuery(value as Record) case 'boolean': return value ? 1 : 0 default: return value } } function convertToPrimitiveValue(value: string) { if (value === '') { return value } if (value === 'true') { return true } if (value === 'false') { return false } const maybeNumber = Number(value) if (!isNaN(maybeNumber)) { return maybeNumber } return value } function queryToFieldValues(query: string) { const caratReplacement = '__CARATREPLACEMENTSTRING__' return query .replaceAll('^^', caratReplacement) .split('^') .filter((term) => term !== 'EQ') .map((term) => term.split('=')) .filter(([, value]) => value !== undefined && value !== '') .map(([key, value]) => ({ [key as string]: convertToPrimitiveValue(value!.replaceAll(caratReplacement, '^')) })) .reduce((acc, obj) => ({ ...acc, ...obj }), {}) } // returning undefined here means the value won't appear on an initial transform export function serializeValueToShape(value: string | undefined, type: string, mandatory: boolean = false) { if (value === undefined) { return value } switch (type) { case 'integer': return value ? parseInt(value, 10) : undefined case 'glide_list': case 'field_list': case 'slushbucket': if (!value) { return mandatory ? [] : undefined } return value.split(',') case 'simple_name_values': if (!value && mandatory) { return {} } return value ? JSON.parse(value) : undefined case 'template_value': return queryToFieldValues(value) case 'boolean': return value === '1' default: return mandatory ? value : value || undefined } }