import { CallExpressionShape, type Factory, Plugin, Shape, type Record, type Diagnostics, type ObjectShape, type RecordId, type NowConfig, DeletedShape, isSNScope, } from '@servicenow/sdk-build-core' import { ModuleFunctionShape } from './server-module-plugin' import { NowIdShape } from './now-id-plugin' import { NowIncludeShape } from './now-include-plugin' import { generateDeprecatedDiagnostics, validateServerScriptField } from './utils' import { RecordPlugin } from './record-plugin' const DEFAULT_MEDIA_TYPE = 'application/json,application/xml,text/xml' const DEFAULT_REST_ENFORCED_ACL = 'cf9d01d3e73003009d6247e603f6a990' const methodsAllowedToOverrideRequests = ['PUT', 'PATCH', 'POST'] const restDefAliases = { serviceId: ['service_id'], enforceAcl: ['enforce_acl'], docLink: ['doc_link'], shortDescription: ['short_description'], protectionPolicy: ['policy'], } const routeAliases = { shortDescription: ['short_description'], requestExample: ['request_example'], enforceAcl: ['enforce_acl'], protectionPolicy: ['policy'], } const versionAliases = { isDefault: ['is_default'], shortDescription: ['short_description'], } const attributeAliases = { shortDescription: ['short_description'], exampleValue: ['example_value'], } function mergeAcls(acls: Shape | undefined) { if (acls?.isArray()) { return acls?.pipe((r) => Array.from(new Set(r.getElements().map((r) => r.ifRecord()?.getId().getValue() ?? r.getValue()))).join(',') ) } return DEFAULT_REST_ENFORCED_ACL } function splitAcls(acls: Shape | undefined) { if (acls?.isString() && !acls?.isEmpty()) { return acls?.pipe((r) => r.getValue().split(',')) } return [] } function getVersionNumber(versionShape: Shape, versions: Record[]) { const versionId = versionShape.ifString()?.getValue() ?? versionShape.ifRecordId()?.getValue() ?? undefined if (!versionId) { return undefined } const version = versions.find((v) => v.getId().getValue() === versionId) if (!version) { return undefined } return version.get('version').toNumber().getValue() } function getAttributeRecord(attrAssociationRecord: Record, attrRecords: Record[], attrType: string) { return attrRecords.find((r) => { const attrId = r.getId().getValue() const attr = attrAssociationRecord.get(`web_service_${attrType}`) const associationId = attr.ifString()?.getValue() ?? attr.asRecordId().getValue() return attrId === associationId }) } function getVersionId( versionShape: Shape, versionToVersionRecordMap: Map, diagnostics: Diagnostics ): string | RecordId { const version = versionShape.ifNumber()?.getValue() ?? undefined if (!version) { return '' } if (!versionToVersionRecordMap.has(`v${version}`)) { diagnostics.error( versionShape, `Unable to resolve version details for version number "${version}". Check if the version is listed in your API versions list.` ) return '' } return versionToVersionRecordMap.get(`v${version}`)!.getId() } async function generateVersionRecords( versions: Shape[], restApi: ObjectShape, restRecord: Record, defaultVersion: number, factory: Factory, diagnostics: Diagnostics, callExpression: CallExpressionShape ): Promise> { const versionMap = new Map() for (const v of versions) { const version = v.asObject().get('version').getValue() const versionId = `v${version}` const versionRecord = await factory.createRecord({ source: callExpression, table: 'sys_ws_version', explicitId: v.asObject().get('$id'), properties: v.asObject().transform(({ $ }) => ({ active: $.def(true), deprecated: $.def(false), is_default: $.val(defaultVersion === version).def(false), short_description: $.from('shortDescription').def(''), version: $, version_id: $.val(versionId), web_service_definition: $.val(restRecord.getId()), })), }) /** versions have to be unique */ if (versionMap.has(versionId)) { diagnostics.error( v.asObject().get('version'), `Duplicate versions with version number "${version}" found in RestApi with id "${restApi.get('$id').getValue()}". All versions are expected to be unique.` ) } versionMap.set(versionId, versionRecord) } return versionMap } function checkForDuplicateRecords( recordMap: Map, record: Record, attrShape: ObjectShape, diagnostics: Diagnostics ): void { const recordId = record.getId().getValue() if (!recordMap.has(recordId)) { recordMap.set(recordId, record) } else { const matchRec = recordMap.get(recordId)! const originalShape = Shape.from(record.getSource(), record.getValue()) const matchedShape = Shape.from(matchRec.getSource(), matchRec.getValue()) if (!matchedShape.equals(originalShape)) { diagnostics.error( attrShape.get('$id'), `Multiple route attributes found with the same ID "${attrShape.get('$id').getValue()}" but with different property values. Attributes with same ID are expected to have identical property values.` ) } } } async function generateRouteAttributeRecords( attributes: Shape[], restDef: Record, routeDef: Record, attrType: string, attrMap: Map, factory: Factory, diagnostics: Diagnostics ): Promise { const records: Record[] = [] for (const attr of attributes) { const attrRecord = await factory.createRecord({ source: attr, table: `sys_ws_${attrType}`, explicitId: attr.asObject().get('$id'), properties: attr.asObject().transform(({ $ }) => ({ name: $, required: $.def(false), short_description: $.from('shortDescription').def(''), example_value: $.from('exampleValue').def(''), web_service_definition: $.val(restDef.getId()), })), installCategory: restDef.getInstallCategory(), }) const isShared = attrMap.has(attrRecord.getId().getValue()) checkForDuplicateRecords(attrMap, attrRecord, attr.asObject(), diagnostics) const attributeMappingRecord = await factory.createRecord({ source: attr, table: `sys_ws_${attrType}_map`, properties: attr.asObject().transform(({ $ }) => ({ web_service_operation: $.val(routeDef.getId()), [`web_service_${attrType}`]: $.val(attrRecord.getId()), })), installCategory: restDef.getInstallCategory(), }) if (isShared) { // The attrRecord was already emitted by a previous route — only emit the mapping records.push(attributeMappingRecord) } else { records.push(attrRecord.with(attributeMappingRecord)) } } return records } async function generateRouteAndRouteAttrRecords( routes: Shape[], restDef: Record, factory: Factory, config: NowConfig, diagnostics: Diagnostics, versionToVersionRecordMap: Map, callExpression: CallExpressionShape ): Promise { const routeRecords: Record[] = [] const headersMap = new Map() const parametersMap = new Map() for (const r of routes) { const route = r.asObject() const routeVersion = route.get('version').getValue() as unknown as number const versionId = getVersionId(route.get('version'), versionToVersionRecordMap, diagnostics) let routeConsumes = route.get('consumes')?.ifString()?.getValue() ?? '' const routeMethod = route.get('method').ifString()?.getValue() ?? 'GET' const restConsumes = restDef.get('consumes').asString().getValue() const restProduces = restDef.get('produces').asString().getValue() if ( routeConsumes && routeConsumes !== restConsumes && !methodsAllowedToOverrideRequests.includes(routeMethod) ) { diagnostics.error(route.get('consumes'), `Cannot override consumer type for ${routeMethod} method`) } routeConsumes = routeConsumes ? routeConsumes : restConsumes const routeProduces = route.get('produces').getValue() ?? restProduces const baseURI = restDef.get('base_uri').getValue() const routePath = route.get('path').getValue() ?? '/' const defaultOperationURI = `${baseURI}${routePath}` const operationURIPrefix = `/api/${restDef.get('namespace').getValue()}` const operationURISuffix = `${restDef.get('service_id').getValue()}${routePath}` const operationURI = versionId ? `${operationURIPrefix}/v${routeVersion}/${operationURISuffix}` : `${operationURIPrefix}/${operationURISuffix}` validateServerScriptField(route.get('script'), diagnostics, config.serverModulesDir) const routeRecord = await factory.createRecord({ source: callExpression, table: 'sys_ws_operation', explicitId: route.get('$id'), properties: route.transform(({ $ }) => ({ name: $.val(route.get('name').ifString()?.getValue() ?? routePath).def('something random'), active: $.def(true), consumes: $.val(routeConsumes), consumes_customized: $.val(routeConsumes !== restConsumes), default_operation_uri: $.val(defaultOperationURI), http_method: $.from('method').def('GET'), operation_script: $.from('script') .map( (v) => v.if(ModuleFunctionShape)?.toString((n) => `${n}({{PARAMS}})`, ['request', 'response']) ?? v ) .toCdata() .def(''), operation_uri: $.val(operationURI), produces: $.val(routeProduces), produces_customized: $.val(routeProduces !== restProduces), relative_path: $.from('path').def('/'), enforce_acl: $.val(mergeAcls(route.get('enforceAcl'))), requires_acl_authorization: $.from('authorization').def(true), requires_authentication: $.from('authentication').def(true), requires_snc_internal_role: $.from('internalRole').def(true), short_description: $.from('shortDescription').def(''), request_example: $.from('requestExample').def(''), web_service_definition: $.val(restDef.getId()), web_service_version: $.val(versionId), sys_policy: $.from('protectionPolicy').def(''), })), }) const routeHeaders = route.get('headers')?.ifArray()?.getElements() ?? [] routeHeaders.forEach((h) => { h.asObject().withAliasedKeys(attributeAliases) generateDeprecatedDiagnostics(h.asObject(), diagnostics) }) const headerRecords = await generateRouteAttributeRecords( routeHeaders, restDef, routeRecord, 'header', headersMap, factory, diagnostics ) const routeParameters = route.get('parameters')?.ifArray()?.getElements() ?? [] routeParameters.forEach((p) => { p.asObject().withAliasedKeys(attributeAliases) generateDeprecatedDiagnostics(p.asObject(), diagnostics) }) const parameterRecords = await generateRouteAttributeRecords( routeParameters, restDef, routeRecord, 'query_parameter', parametersMap, factory, diagnostics ) routeRecords.push(routeRecord.with(...headerRecords, ...parameterRecords)) } return routeRecords } function findReclaimedRecords(records: Record[], orphanedRecords: Record[]) { return records .filter( (record) => record.getCreator()?.getName() === RecordPlugin.getName() && !orphanedRecords.find((orphan) => orphan.getId().getValue() === record.getId().getValue()) ) .map((record) => new DeletedShape({ source: record.getSource() })) } function findOrphanedAttributes( attributes: Record[], attributeRouteAssociations: Record[], attributeType: string ): Record[] { const attributeIdToRecordMap = new Map() attributes.forEach((attribute) => { attributeIdToRecordMap.set(attribute.getId().getValue(), attribute) }) const attributeIds = new Set(attributeIdToRecordMap.keys()) const referencedAttributeIds = new Set( attributeRouteAssociations.map((m2m) => { const attr = m2m.get(`web_service_${attributeType}`) return attr.ifString()?.getValue() ?? attr.ifRecordId()?.getValue() ?? undefined }) ) const orphanedAttributeIds = Array.from(attributeIds).filter((id) => !referencedAttributeIds.has(id)) return orphanedAttributeIds .map((id) => attributeIdToRecordMap.get(id)) .filter((record): record is Record => record !== undefined) } function validateAttributesAndAssociations( attributes: Record[], attributeRouteAssociations: Record[], attributeType: string ) { const attributeIdToRecordMap = new Map() attributes.forEach((attribute) => { attributeIdToRecordMap.set(attribute.getId().getValue(), attribute) }) const invalidAttributeAssociations = attributeRouteAssociations.filter((association) => { const attr = association.get(`web_service_${attributeType}`) const attributeId = attr.ifString()?.getValue() ?? attr.ifRecordId()?.getValue() ?? '' return !attributeIdToRecordMap.has(attributeId) }) if (invalidAttributeAssociations.length > 0) { const invalidAttributeAssociationIds = invalidAttributeAssociations .map((association) => association.getId().getValue()) .join(', ') throw new Error( `Found ${invalidAttributeAssociations.length} ${attributeType} association record(s) referencing non-existent ${attributeType}s: ${invalidAttributeAssociationIds}. ` + `Please clean up these m2m records.` ) } } function routeAttributeTransform( associationRecords: Record[], attributes: Record[], operationFilterId: string, attributeType: string ) { const results = associationRecords .filter((pm) => { const operation = pm.get(`web_service_operation`) const operationId = operation.asRecordId().getValue() return operationId === operationFilterId }) .map((pm) => { const attribute = getAttributeRecord(pm, attributes, attributeType)! return attribute .transform(({ $ }) => ({ $id: $.val(NowIdShape.from(attribute)), name: $, required: $.toBoolean().def(false), exampleValue: $.from('example_value').def(''), shortDescription: $.from('short_description').def(''), })) .withAliasedKeys(attributeAliases) }) return results } function versionsTransform(versions: Record[]) { if (versions.length === 0) { return undefined } return versions.map((v) => v .transform(({ $ }) => ({ $id: $.val(NowIdShape.from(v)), active: $.toBoolean().def(true), deprecated: $.toBoolean().def(false), isDefault: $.from('is_default').toBoolean().def(false), shortDescription: $.from('short_description').def(''), version: $.toNumber(), })) .withAliasedKeys(versionAliases) ) } export const RestApiPlugin = Plugin.create({ name: 'RestApiPlugin', records: { sys_ws_header_map: { coalesce: ['web_service_operation', 'web_service_header'], }, sys_ws_query_parameter_map: { coalesce: ['web_service_operation', 'web_service_query_parameter'], }, sys_ws_definition: { relationships: { sys_ws_operation: { via: 'web_service_definition', descendant: true, relationships: { sys_ws_header_map: { via: 'web_service_operation', descendant: true, }, sys_ws_query_parameter_map: { via: 'web_service_operation', descendant: true, }, }, }, sys_ws_version: { via: 'web_service_definition', descendant: true, }, sys_ws_header: { via: 'web_service_definition', descendant: true, }, sys_ws_query_parameter: { via: 'web_service_definition', descendant: true, }, }, async toShape(record, { descendants, transform, config }) { const versions = descendants.query('sys_ws_version') const routes = descendants.query('sys_ws_operation') const headers = descendants.query('sys_ws_header') const parameters = descendants.query('sys_ws_query_parameter') const headerRouteAssociations = descendants.query('sys_ws_header_map') const paramRouteAssociations = descendants.query('sys_ws_query_parameter_map') /** * Special cases: * Case 1: No m2m records exists for a header or parameter record. Can happen in following scenarios: * - User has created a header or parameter record but these records are not associated with any routes on instance. * - User has removed all the m2m associations on instance but not the actual header or parameter record. * Solution: Return orphaned attributes as unhandled records for partial success * * Case 2: Deleting a header record on instance wouldn't delete the m2m records which map those headers to a route. * Solution: Throw error and request the user to cleanup the m2m records on instance. */ validateAttributesAndAssociations(headers, headerRouteAssociations, 'header') validateAttributesAndAssociations(parameters, paramRouteAssociations, 'query_parameter') const orphanedHeaders = findOrphanedAttributes(headers, headerRouteAssociations, 'header') const orphanedParameters = findOrphanedAttributes(parameters, paramRouteAssociations, 'query_parameter') const unhandledRecords = [...orphanedHeaders, ...orphanedParameters] const routesWithScript = await Promise.all( routes.map(async (r) => { const script = await NowIncludeShape.fromRecord(r, r.get('operation_script'), transform) return r .transform(({ $ }) => ({ $id: $.val(NowIdShape.from(r)), name: $, active: $.toBoolean().def(true), consumes: $, method: $.from('http_method').def('GET'), script: $.val(script), produces: $, path: $.from('relative_path').def('/'), enforceAcl: $.val(splitAcls(r.get('enforce_acl'))).def([DEFAULT_REST_ENFORCED_ACL]), authorization: $.from('requires_acl_authorization').toBoolean().def(true), authentication: $.from('requires_authentication').toBoolean().def(true), internalRole: $.from('requires_snc_internal_role').toBoolean().def(true), shortDescription: $.from('short_description').def(''), requestExample: $.from('request_example').def(''), protectionPolicy: $.from('sys_policy').def(''), parameters: $.val( routeAttributeTransform( paramRouteAssociations, parameters, r.getId().getValue(), 'query_parameter' ) ).def([]), headers: $.val( routeAttributeTransform( headerRouteAssociations, headers, r.getId().getValue(), 'header' ) ).def([]), version: $.from('web_service_version').map((v) => getVersionNumber(v, versions)), })) .withAliasedKeys(routeAliases) }) ) const shape = new CallExpressionShape({ source: record, callee: 'RestApi', args: [ record .transform(({ $ }) => ({ $id: $.val(NowIdShape.from(record)), name: $, active: $.toBoolean().def(true), consumes: $.def(DEFAULT_MEDIA_TYPE), produces: $.def(DEFAULT_MEDIA_TYPE), enforceAcl: $.from('enforce_acl') .map((v) => splitAcls(v)) .def([DEFAULT_REST_ENFORCED_ACL]), serviceId: $.from('service_id'), namespace: $.def(config.scope), shortDescription: $.from('short_description').def(''), protectionPolicy: $.from('sys_policy').def(''), docLink: $.from('doc_link').def(''), routes: $.val(routesWithScript).def([]), versions: $.val(versionsTransform(versions)).def([]), })) .withAliasedKeys(restDefAliases), ], }) const reclaimedRecords = [ ...findReclaimedRecords(headers, orphanedHeaders), ...findReclaimedRecords(parameters, orphanedParameters), ] if (unhandledRecords.length > 0) { return { success: 'partial', value: shape, unhandledRecords, reclaimedRecords, } } return { success: true, value: shape, reclaimedRecords, } }, }, }, shapes: [ { shape: CallExpressionShape, fileTypes: ['fluent'], async toRecord(callExpression, { factory, config, diagnostics }) { if (callExpression.getCallee() !== 'RestApi') { return { success: false } } const restApi = callExpression.getArgument(0).asObject().withAliasedKeys(restDefAliases) generateDeprecatedDiagnostics(restApi, diagnostics) const scope = config.scope const userNamespace = restApi.get('namespace').ifString()?.getValue() const namespace = isSNScope(scope) && userNamespace ? userNamespace : scope if (!isSNScope(scope) && userNamespace && userNamespace !== scope) { diagnostics.hint( restApi.get('namespace'), `Custom namespace is only supported for ServiceNow scoped applications. Using scope '${scope}' as namespace.` ) } const serviceId = restApi.get('serviceId') const serviceIdRegex = /^[a-z0-9_]*[a-z0-9]$/ if (!serviceId.asString().getValue().match(serviceIdRegex)) { diagnostics.error( serviceId, `Rest service_id must only contain lowercase letters, numbers, and underscores and end with a letter or number` ) } const versions = restApi.get('versions')?.ifArray()?.getElements() ?? [] versions.forEach((v) => { v.asObject().withAliasedKeys(versionAliases) generateDeprecatedDiagnostics(v.asObject(), diagnostics) }) const defaultVersions = versions.filter( (v) => v.asObject().get('isDefault').ifBoolean()?.getValue() ?? false ) const defaultVersion = defaultVersions.length === 1 ? defaultVersions[0]!.asObject().get('version').getValue() : -1 if (defaultVersions.length > 1) { diagnostics.error(restApi.get('versions'), `Multiple versions cannot be set to default.`) } const enforceAcls = restApi.get('enforceAcl') const restRecord = await factory.createRecord({ source: callExpression, table: 'sys_ws_definition', explicitId: restApi.get('$id'), properties: restApi.transform(({ $ }) => ({ name: $, service_id: $.val(serviceId), active: $.def(true), base_uri: $.val(`/api/${namespace}/${serviceId.getValue()}`), short_description: $.from('shortDescription').def(''), consumes: $.def(DEFAULT_MEDIA_TYPE), consumes_customized: $.from('consumes').map((v) => v.isString() ? v.getValue() !== DEFAULT_MEDIA_TYPE : false ), default_version: $.val( defaultVersion !== -1 ? `v${defaultVersion}` : 'No active default version' ), is_versioned: $.val(versions.length > 0 ? true : false), namespace: $.val(namespace), doc_link: $.from('docLink').def(''), produces: $.def(DEFAULT_MEDIA_TYPE), produces_customized: $.from('produces').map((v) => v.isString() ? v.getValue() !== DEFAULT_MEDIA_TYPE : false ), sys_policy: $.from('protectionPolicy').def(''), enforce_acl: $.val(mergeAcls(enforceAcls)), })), }) const versionToVersionRecordMap = await generateVersionRecords( versions, restApi, restRecord, defaultVersion as number, factory, diagnostics, callExpression ) const routes = restApi.get('routes')?.ifArray()?.getElements() ?? [] routes.forEach((r) => { r.asObject().withAliasedKeys(routeAliases) generateDeprecatedDiagnostics(r.asObject(), diagnostics) }) const routeAndRouteAttrRecords = await generateRouteAndRouteAttrRecords( routes, restRecord, factory, config, diagnostics, versionToVersionRecordMap, callExpression ) return { success: true, value: restRecord.with(...versionToVersionRecordMap.values(), ...routeAndRouteAttrRecords), } }, }, ], })