import { normalizeName } from './profileGeneratorHelper'; import inquirer from 'inquirer'; import { findEntityByIdOrName, loadCached } from './entityHelper'; import { readVariableKey } from './variablesHelper'; import { warningSymbol } from './log'; import chalk from 'chalk'; import { toJsonObject } from 'curlconverter'; import fs from 'fs'; export const REST_API_DATA_TEMPLATE = { url: '/api/example', type: 'REST', method: 'POST', headers: { 'Content-Type': ['application/json'], }, queries: {}, body: '', acceptHttpStatuses: { success: { parser: 'regex', expression: '2\\d{2}', }, }, responseFilters: { jsonata: { items: { acceptedStatuses: ['success'], expression: `$.results.($i := $; properties.{ "id": $i.id, "email": email })[]`, }, total: { acceptedStatuses: ['success'], expression: `$.total`, }, }, scriban: { hasNextPage: { acceptedStatuses: ['success'], expression: `{{ (__headers.page_info | string.size) > 0 }}`, }, }, }, }; export const GRAPHQL_API_DATA_TEMPLATE = { url: '/api/example', type: 'GRAPHQL', method: 'POST', headers: { 'Content-Type': ['application/json'], }, queries: {}, body: { query: '', variables: {}, }, acceptHttpStatuses: { success: { parser: 'regex', expression: '2\\d{2}', }, }, responseFilters: { jsonata: { items: { acceptedStatuses: ['success'], expression: `$.products.($product := $; variants.{ "product_id": $string($product.id), "variant_id": $string(id) })[]`, }, }, scriban: { hasNextPage: { acceptedStatuses: ['success'], expression: `{{ __headers.page_info | string.size > 0 }}`, }, }, }, }; function normalizeLineBreaks(obj: any): any { if (obj === null || obj === undefined) { return obj; } if (typeof obj === 'string') { return obj.replace(/[\r\n]+/g, '\n'); } if (typeof obj === 'object' && !Array.isArray(obj)) { for (const key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { obj[key] = normalizeLineBreaks(obj[key]); } } } else if (Array.isArray(obj)) { return obj.map(normalizeLineBreaks); } return obj; } export function createAPITemplate(apiType: string, sampleJson: any) { if (!sampleJson) { return apiType === 'GRAPHQL' ? GRAPHQL_API_DATA_TEMPLATE : REST_API_DATA_TEMPLATE; } // Normalize line breaks in sampleJson sampleJson = normalizeLineBreaks(sampleJson); // loop through sampleJson's properties recursively // if the value is not null and is a string, => apply .replace(/[\r\n]+/g, '\n') // Convert headers to Dictionary const formattedHeaders = Object.entries(sampleJson.headers || {}).reduce( (acc, [key, value]) => { acc[key] = (Array.isArray(value) ? value : [value]) as any; return acc; }, {} as Record, ); if (apiType === 'GRAPHQL') { return { ...GRAPHQL_API_DATA_TEMPLATE, url: sampleJson.url || GRAPHQL_API_DATA_TEMPLATE.url, method: sampleJson.method?.toUpperCase() || GRAPHQL_API_DATA_TEMPLATE.method, headers: formattedHeaders, body: { query: sampleJson.data?.query?.toString()?.replace(/[\r\n]+/g, '\n') || GRAPHQL_API_DATA_TEMPLATE.body.query, variables: sampleJson.data?.variables || GRAPHQL_API_DATA_TEMPLATE.body.variables, }, }; } return { ...REST_API_DATA_TEMPLATE, url: sampleJson.url || REST_API_DATA_TEMPLATE.url, method: sampleJson.method?.toUpperCase() || REST_API_DATA_TEMPLATE.method, headers: formattedHeaders, queries: sampleJson.queries || REST_API_DATA_TEMPLATE.queries, body: sampleJson.data ? JSON.stringify(sampleJson.data, null, 2) : REST_API_DATA_TEMPLATE.body, // Format JSON with indentation }; } export async function createApiStep( stepName: string, options?: { sample?: string }, ) { var sample = options?.sample?.trim(); let apiType = ''; if (!!sample) { if (!sample?.toLowerCase().startsWith('curl ')) { // Check if sample is a path to a file if (fs.existsSync(sample)) { sample = fs.readFileSync(sample, 'utf8')?.trim(); } } if (!sample?.toLowerCase().startsWith('curl ')) { console.error( chalk.red( `✗ The given sample ${sample} is not a valid curl command or file path.`, ), ); // return apiType === "GRAPHQL" ? GRAPHQL_API_DATA_TEMPLATE : REST_API_DATA_TEMPLATE; return null; } } var sampleJson = !!sample ? toJsonObject(sample) : null; var isGraphQL = ['graphql.json'].find((f) => sampleJson?.url?.endsWith(f)) || sampleJson?.data?.hasOwnProperty('query'); if (isGraphQL) { apiType = 'GRAPHQL'; } else if (!!sampleJson?.url || !!sampleJson?.data) { apiType = 'REST'; } else { let { selectedAPIType } = await inquirer.prompt([ { type: 'list', name: 'selectedAPIType', message: 'Which type of API would you like to add?', choices: ['REST', 'GRAPHQL', 'GOOGLE_APP_SCRIPT'], }, ]); apiType = selectedAPIType || 'REST'; } var createdApiTemplate = createAPITemplate(apiType, sampleJson); var connections = await loadCached('connection'); // ask for which connection to use using inquirer with connections const { selectedConnection } = await inquirer.prompt([ { type: 'list', name: 'selectedConnection', message: 'Which connection would you like to use?', choices: connections.map((c) => ({ name: c.entityName, value: c })), }, ]); var normalizedStepName = normalizeName(stepName); var stepTemplate = { key: normalizedStepName.underscoreUpperCase, rootStep: false, name: normalizedStepName.displayName, description: '', connectorId: readVariableKey(selectedConnection?.entityId), displayType: 'API', next: `{{ if (pusher.current.${normalizedStepName.underscoreDisplayName}?.id | string.size) > 0 "CREATE" else "UPDATE" end }}`, arguments: { limit: 100, updated_at: "{{(puller.history.GET_LAST_UPDATED?.updated_at | string.size) > ? puller.history.GET_LAST_UPDATED.updated_at : ''}}", after: `{{(puller.lastToken?.${normalizedStepName.underscoreUpperCase}?.after | string.size) > 0 ? puller.lastToken.${normalizedStepName.underscoreUpperCase}.after : 0}}`, }, data: createdApiTemplate, }; console.log( chalk.whiteBright(` ${warningSymbol()} ${chalk.bold.yellow(`Important:`)} After the step is created, please follow these instructions: 1. Change the ${chalk.bold.yellow('data.url')} to match the endpoint of the connected system. 2. For ${chalk.bold.yellow('REST API')}, update the ${chalk.bold.yellow('method')}, ${chalk.bold.yellow('queries')}, and ${chalk.bold.yellow('body')} as needed. 3. Write an appropriate ${chalk.bold.yellow('jsonata expression')} to parse the response from the API request. You can test and validate the response using tools like ${chalk.green('POSTMAN')} or ${chalk.green('https://try.jsonata.com')}. 4. If you need to handle exceptions: 4.1 Add a new ${chalk.bold.yellow('HttpStatusCode')} to ${chalk.bold.yellow('acceptedHttpStatuses')}, e.g.: ${chalk.green(`bad_request: { parser: 'regex', expression: '400' }`)} 4.2 Query for the error message using ${chalk.bold.yellow('jsonata')} in ${chalk.bold.yellow('responseFilters')} (if applicable). 5. Ensure that the ${chalk.bold.yellow('responseFilters')} are correctly configured to extract the required data and metadata (e.g., ${chalk.bold.yellow("'total'")}, ${chalk.bold.yellow("'nextPageInfo'")}). 6. Use ${chalk.bold.yellow('scriban')} to process the response further if needed. For example: ${chalk.green('{{(filters.jsonata.total | array.size) > 0}}')} or ${chalk.green('{{ (__headers.page_info | string.size) > 0 }}')}. 7. Verify that the ${chalk.bold.yellow('arguments')} are properly set up to use data from ${chalk.bold.yellow('beforePullSteps')} or ${chalk.bold.yellow('beforePushSteps')}. For example: ${chalk.green("{{(puller.history.GET_LAST_UPDATED?.updated_at | string.size) > ? puller.history.GET_LAST_UPDATED.updated_at : ''}}")} (Assume that there is a step with the key ${chalk.bold.yellow('GET_LAST_UPDATED')}). `), ); return stepTemplate; } export async function createSqlStep( stepName: string, options?: { sample?: string }, ) { const { queryType } = await inquirer.prompt([ { type: 'list', name: 'queryType', message: 'What type of query would you like to create?', choices: [ { name: 'Query for a list of data', value: 'list' }, { name: 'Query for the last updated time', value: 'lastUpdated' }, ], }, ]); const { taskName } = await inquirer.prompt([ { type: 'input', name: 'taskName', message: 'Enter the name of the task you would like to select from:', }, ]); const entity = await findEntityByIdOrName('', taskName, 'task'); if (!entity) { console.error(`${chalk.red('✗')} Task with name "${taskName}" not found.`); return null; } const valueTableNameKey = readVariableKey(entity.content.valueTableName); var normalizedStepName = normalizeName(stepName); var stepTemplate = { key: normalizedStepName.underscoreUpperCase, rootStep: false, name: normalizedStepName.displayName, description: '', connectorId: '**SelfConnectorId**', displayType: 'SQL', next: `{{ if (pusher.current.${normalizedStepName.underscoreDisplayName}?.id | string.size) > 0 "CREATE" else "UPDATE" end }}`, arguments: {}, data: { timeout: '60', sql: queryType === 'list' ? `select * from ${valueTableNameKey} limit 100` : `SELECT TO_CHAR(_.updated_at::timestamptz - INTERVAL '1 DAY', 'YYYY-MM-DD') AS updated_at FROM ${valueTableNameKey} _ ORDER BY _.updated_at DESC NULLS LAST LIMIT 1`, responseFilters: [ { key: queryType === 'list' ? 'items' : 'updated_at', type: queryType === 'list' ? 'List' : 'Field', }, ], hasNextPage: null as any, }, }; if (queryType === 'list') { stepTemplate.data.hasNextPage = 'items'; } console.log( chalk.whiteBright(` ${warningSymbol()} ${chalk.bold.yellow('Important:')} After the step is created, please follow these instructions: 1. Replace the ${chalk.bold.yellow('sql')} query with the correct query for your use case. 2. If you need to join another task, use ${chalk.bold.yellow("hexasync task files -n ''")} to search for the ${chalk.bold.yellow('valueTableName')} to join. Use the key variable (e.g., ${chalk.bold.yellow('**Product_Table_Name**')}) for best practices in HexaSync Template. 3. Depending on the query purpose, it might return ${chalk.bold.yellow('List')}, ${chalk.bold.yellow('Row')}, or ${chalk.bold.yellow('Field')}. Please update the ${chalk.bold.yellow('responseFilters')} to ensure correct results. 4. For the puller, the ${chalk.bold.yellow('hasNextPage')} is the key for HexaSync to determine if the polling action succeeded. `), ); return stepTemplate; } export function createForwardStep( stepName: string, options?: { sample?: string }, ) { var normalizedStepName = normalizeName(stepName); var stepTemplate = { key: normalizedStepName.underscoreUpperCase, rootStep: true, name: normalizedStepName.displayName, description: '', connectorId: '**SelfConnectorId**', displayType: 'FORWARD', next: `{{ if (item.__destination_id | string.size) == 0 if (item | hexasync.item.should_remove) != true "CHECK_EXISTS" end else if (["[ignored]", "n/a"] | array.contains item.__destination_id) != true if (item | hexasync.item.should_remove) != true "UPDATE" else "REMOVE" end end }}`, arguments: {}, }; return stepTemplate; }