/* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import _ from 'lodash'; import parseLinkHeader from 'parse-link-header'; import { Agent, setGlobalDispatcher } from 'undici'; import { ApiInParameterSpec, ApiOutParameterSpec, ApiSpec, AuthSpec, StatusSpec } from '../schema/WebApiSchema.js'; import { PagingState } from '../types/PagingState.js'; import { PluginContext } from '../types/PluginContext.js'; import { SqupContext } from '../types/SqupContext.js'; import type { } from '../types/fetch.d.ts'; // NodeJS v18 has fetch in core, but no types :( import { PerformCalculation } from './Calculations.js'; import { EvaluateCondition } from './EvaluateCondition.js'; import { getArray } from './GetArray.js'; import { processOAuth2 } from './OAuth2.js'; import { Render, RenderObject } from './Render.js'; import { GetPageSize, mergeBodies, NormalizeNextUrl, timeArrived } from './Util.js'; import { parseStringPromise } from 'xml2js'; export async function CallApi( pluginContext: PluginContext, apiIn: ApiSpec, pagingState: PagingState, squpContext: SqupContext, replacements?: Record> ): Promise<{ response?: Response, data: any }> { const api = getApi(pluginContext, apiIn, squpContext); const headers = Object.entries(api.headers ?? {}).reduce( (acc, [key, val]) => ({ ...acc, [key]: Render(val.toString(), pluginContext, replacements) }), {} as Record); const queryArgs = Object.entries(api.queryArgs ?? {}).reduce( (acc, [key, val]) => ({ ...acc, [key]: Render(val.toString(), pluginContext, replacements) }), {} as Record); const body = RenderBody(api, pluginContext, squpContext, replacements ?? {}); let pageSize; let nextUrl = ''; if (api.pagingModel === 'nextUrl' && pagingState.nextUrl) { const nextUrlObj = new URL(pagingState.nextUrl); nextUrl = `${nextUrlObj.protocol}//${nextUrlObj.host}${nextUrlObj.pathname}`; for (const [key, val] of Array.from(nextUrlObj.searchParams.entries())) { queryArgs[key] = val; } } const addInParameter = (param: ApiInParameterSpec, value: string) => { switch (param.mode) { case 'header': { headers[param.path] = value; break; } case 'queryArg': { queryArgs[param.path] = value; break; } default: throw new Error(`Bad parameter mode: ${param.mode}`); } }; let callStillNeeded = true; let retryForThrottling = false; let attempt = 1; // Loop to allow token refresh while (callStillNeeded || retryForThrottling) { retryForThrottling = false; if (api.auth) { const auth = renderAuth(api.auth, pluginContext, replacements); switch (auth.type) { case 'bearerToken': { addInParameter(auth, auth.token ?? ''); callStillNeeded = false; break; } case 'basicUserPassword': { const value = `Basic ${Buffer.from(`${auth.username}:${auth.password}`).toString('base64')}`; addInParameter(auth, value); callStillNeeded = false; break; } case 'oauth2': { const authQueryArgs = await processOAuth2( headers, attempt > 1, //refresh auth, pluginContext, squpContext); authQueryArgs.forEach(({ key, value }) => (queryArgs[key] = value)); break; } default: throw new Error(`Bad auth type: ${auth.type}`); } } else { callStillNeeded = false; } // set up paging params if (api.pageSizeIn) { pageSize = GetPageSize(pluginContext, squpContext, api.pageSizeIn); addInParameter(api.pageSizeIn, pageSize.toString()); } switch (api.pagingModel) { case 'nextUrl': { break; } case 'token': { if (api.nextTokenIn && pagingState.pageToken) { addInParameter(api.nextTokenIn, pagingState.pageToken); } break; } case 'offset': { if (api.nextOffsetIn) { addInParameter(api.nextOffsetIn, pagingState.recordCount.toString()); } break; } default: { // No paging } } const queryArgsString = Object.entries(queryArgs).length > 0 ? `?${new URLSearchParams(queryArgs)}` : ''; const url = api.pagingModel === 'nextUrl' && pagingState.nextUrl ? nextUrl + queryArgsString : Render(api.url!, pluginContext, replacements) + queryArgsString; try { let payload: any; let data: any; const bodyString = api.bodyType === 'x-www-form-urlencoded' ? new URLSearchParams(body).toString() : JSON.stringify(body); headers['Accept'] = 'application/json'; // Check for ignoreCertificateErrors in plugin config if ((pluginContext.config?.pluginConfig as any)['ignoreCertificateErrors']) { const agent = new Agent({ connect: { rejectUnauthorized: false } }); setGlobalDispatcher(agent); } if (pluginContext.retryAfter) { // We have been instructed to wait before making the next call due to throttling if (typeof squpContext.canContinue === 'function') { const delayMsecs = Math.max(pluginContext.retryAfter.getTime() - new Date().getTime(), 0); if (!squpContext.canContinue(delayMsecs)) { // Bad news: we're constrained for time and we can't make the next call in time! squpContext.report.error('Rate limiting prevented retrieval of data'); } } squpContext.log.debug(`**********${new Date().toISOString()}: waiting due to throttling until ${pluginContext.retryAfter} to fetch from ${url} - bodyString: ${bodyString}`); await timeArrived(pluginContext.retryAfter); } squpContext.log.debug(`**********${new Date().toISOString()}: about to fetch from ${url} - bodyString: ${bodyString}`); const response = await fetch(url, { method: api.httpMethod, body: bodyString, headers }); squpContext.log.debug(`**********${new Date().toISOString()}: ${response.status} from ${url} - bodyString: ${bodyString}`); const getOutParameter = (param: ApiOutParameterSpec) => { switch (param.mode) { case 'webLink': { const parsedLinks = parseLinkHeader(response.headers?.get('link')); const link = parsedLinks?.[param.path]; return link ? link.url : link; } case 'header': { return response.headers?.get(param.path); } case 'payload': { return getArray(payload, param.path); } default: throw new Error(`Bad parameter mode: ${param.mode}`); } }; if (response.status >= 200 && response.status < 300) { // Success callStillNeeded = false; payload = await response.json(); if (api.xmlPayload) { try { payload = await parseStringPromise(payload) } catch (err: any) { squpContext.log.debug(`xml parsing error: ${err.message}`); } } if (Array.isArray(api.status) && api.status.length > 0) { // Get any retry time information const retryAfter = api.retryAfter ? getOutParameter(api.retryAfter) : undefined; // Check status per webapi spec const statusReplacements = { ...replacements, response, payload, retryAfter }; for (const check of api.status) { const status = await CheckStatus(pluginContext, check, retryAfter, api.retryAfter?.expression, squpContext, statusReplacements); if (status === 'Retry') { retryForThrottling = true; break; } } } if (!retryForThrottling) { data = api.dataPath ? getArray(payload, api.dataPath) : payload; squpContext.log.debug('data'); if (Array.isArray(data)) { squpContext.log.debug(`received array with ${data.length}`); //squpContext.log.debug(JSON.stringify(data, null, 4)); pagingState.recordCount += data.length; } else { squpContext.log.debug('received scalar'); //squpContext.log.debug(JSON.stringify(data, null, 4)); pagingState.recordCount++; } if (api.pagingModel) { if (api.nextUrlOut) { const nextUrl = getOutParameter(api.nextUrlOut); if (nextUrl) { const baseUrl = api.pagingOptions?.baseUrlTemplate ? Render(api.pagingOptions.baseUrlTemplate, pluginContext) : new URL(url).origin; const normalizedNextUrl = NormalizeNextUrl(nextUrl, baseUrl); if (pagingState.nextUrl === normalizedNextUrl) { // Some APIs (erroneously?) return a repeated nextUrl value at the end of the data set pagingState.complete = true; pagingState.nextUrl = undefined; } else { pagingState.nextUrl = normalizedNextUrl; } } else { pagingState.complete = true; pagingState.nextUrl = undefined; } } else if (api.nextTokenOut) { pagingState.pageToken = getOutParameter(api.nextTokenOut); if (!pagingState.pageToken) { pagingState.complete = true; } } else if (api.pagingModel === 'offset') { if (!pageSize) { throw new Error('When pagingModel is "offset", pageSizeIn must be defined and non-zero'); } pagingState.complete = !Array.isArray(data) || data.length < pageSize; } } else { pagingState.complete = true; } } } else { squpContext.log.debug(`fetch error status - status: ${response.status} - ${response.statusText}`); if (Array.isArray(api.status) && api.status.length > 0) { // Get any retry time information const retryAfter = api.retryAfter ? getOutParameter(api.retryAfter) : undefined; // Check status per webapi spec const statusReplacements = { ...replacements, response, retryAfter }; for (const check of api.status) { const status = await CheckStatus(pluginContext, check, retryAfter, api.retryAfter?.expression, squpContext, statusReplacements); if (status === 'Retry') { retryForThrottling = true; break; } } } } if (!retryForThrottling) { if (!callStillNeeded && api.auth?.type === 'oauth2' && attempt < 2) { callStillNeeded = true; } if (!callStillNeeded) { return { response, data }; } } } catch (err: any) { squpContext.log.debug(`fetch exception: ${err.message}: ${err.cause?.message}`); if (api.auth?.type === 'oauth2' && attempt < 2) { callStillNeeded = true; } else { if (err.cause?.message) { squpContext.report.error(err.cause.message); } throw err; } } if (!retryForThrottling) { attempt++; } } throw new Error('Never gets here'); } function RenderBody( api: ApiSpec, pluginContext: PluginContext, squpContext: SqupContext, replacements: Record> ) { if (!api.body) { return undefined; } let mergedBody = { ...api.body }; if (Array.isArray(api.bodies)) { for (const body of api.bodies) { const condition = EvaluateCondition(pluginContext, squpContext, body.condition, replacements); if (condition) { mergedBody = mergeBodies(mergedBody, body.body); if (body.break) { break; } } } } return RenderObject(mergedBody, pluginContext, replacements); } function renderAuth( auth: AuthSpec, pluginContext: PluginContext, replacements?: Record> ): AuthSpec { const result = { ...auth, token: auth.token ? Render(auth.token ?? '', pluginContext, replacements) : undefined, username: auth.username ? Render(auth.username ?? '', pluginContext, replacements) : undefined, password: auth.password ? Render(auth.password ?? '', pluginContext, replacements) : undefined, oauth2TokenUrl: auth.oauth2TokenUrl ? Render(auth.oauth2TokenUrl ?? '', pluginContext, replacements) : undefined, oauth2ClientId: auth.oauth2ClientId ? Render(auth.oauth2ClientId ?? '', pluginContext, replacements) : undefined, oauth2ClientSecret: auth.oauth2ClientSecret ? Render(auth.oauth2ClientSecret ?? '', pluginContext, replacements) : undefined }; return result; } async function CheckStatus( pluginContext: PluginContext, check: StatusSpec, retryAfter: any, retryAfterExpression: string | undefined, squpContext: SqupContext, replacements: Record> ): Promise<'Continue' | 'Retry'> { const condition = EvaluateCondition(pluginContext, squpContext, check.condition, replacements); if (condition) { const message = check.message ? Render(check.message, pluginContext, replacements) : check.message; switch (check.result) { case 'error': { const errMsg = `${check.name}: ${message}`; squpContext.report.error(errMsg); throw new Error(errMsg); } case 'warning': { const msg = `${check.name}: ${message}`; squpContext.report.warning(msg); return 'Continue'; } case 'throttled': { const now = new Date(); const nowMsecs = now.getTime(); if (typeof retryAfter === 'string') { const retryAfterNumber = Number(retryAfter); if (isNaN(retryAfterNumber)) { // The retry-after value is a non-numeric string - we assume this is an HTML syntax date. const retryDate = new Date(retryAfter); // The Date constructor handles HTML syntax dates OK if (isNaN(retryDate.valueOf())) { squpContext.log.debug(`Throttled at ${now}, string retryAfter=${retryAfter}: **unparsable** not setting retry after time`); } else { squpContext.log.debug(`Throttled at ${now}, string retryAfter="${retryAfter}", parsable string setting retry after time to ${retryDate}`); pluginContext.retryAfter = retryDate; } } else { // The retry-after value is a numeric string - run any supplied calculation or assume it's a number of seconds to delay. const retryAfterMsecs = retryAfterExpression ? PerformCalculation(pluginContext, squpContext, retryAfterExpression, { ...replacements, retryAfter }) : retryAfterNumber * 1000; const when = new Date(nowMsecs + retryAfterMsecs); squpContext.log.debug(`Throttled at ${now}, string retryAfter="${retryAfter}", retryAfterMsecs="${retryAfterMsecs}" - setting retry after time to ${when}`); pluginContext.retryAfter = when; } } else if (typeof retryAfter === 'number') { const retryAfterMsecs = retryAfterExpression ? PerformCalculation(pluginContext, squpContext, retryAfterExpression, { ...replacements, retryAfter }) : retryAfter * 1000; const when = new Date(nowMsecs + retryAfterMsecs); squpContext.log.debug(`Throttled at ${now}, numeric retryAfter="${retryAfter}", retryAfterMsecs="${retryAfterMsecs}" - setting retry after time to ${when}`); pluginContext.retryAfter = when; } else { squpContext.log.debug(`Throttled at ${now}, bad retryAfter type="${typeof retryAfter}", not setting retry after time`); } const msg = `${check.name}: ${message ?? 'The API is throttling requests'}`; return 'Retry'; } default: { throw new Error(`Unexpected status check result type: ${check.result}`); } } } return 'Continue'; } function getApi(pluginContext: PluginContext, apiIn: ApiSpec, squpContext: SqupContext): ApiSpec { if (!apiIn.extends) { return apiIn; } const baseApi = pluginContext.apiDefinitions.get(apiIn.extends); if (!baseApi) { throw new Error(`api spec extends non-existent api definition: "${apiIn.extends}"`); } const result = _.merge( {}, baseApi, apiIn); const statusArray = [ ...(Array.isArray(baseApi.status) ? baseApi.status : []), ...(Array.isArray(apiIn.status) ? apiIn.status : []) ]; result.status = statusArray; return result; }