import { Command } from 'commander'; import path from 'path'; import { includeAll } from './includeFiles/includeAll'; import { includeTask } from './includeFiles/includeTask'; import { includeConnection } from './includeFiles/includeConnection'; import { includeReport } from './includeFiles/includeReport'; import { includeSchema } from './includeFiles/includeSchema'; import { includePusher } from './includeFiles/includePusher'; import { includeDependency } from './includeFiles/includeDependency'; import { includeTransformation } from './includeFiles/includeTransformation'; import { includeValidation } from './includeFiles/includeValidation'; import { globSync } from 'glob'; import Fuse from 'fuse.js'; import inquirer from 'inquirer'; import fs from 'fs'; import { parse, stringify } from 'yaml'; import { generateAssociation, includeYaml, } from '../../helpers/profileGeneratorHelper'; import { getProjectPath } from '../../helpers/cluserHelper'; import { importCommon } from './includeFiles/importCommonFiles'; import { paddingList } from '../../helpers/paddingList'; import { v4 } from 'uuid'; import { transferVariables } from '../../helpers/variablesHelper'; import { IExternalReplacement } from '../../helpers/@types/IExternalReplacement'; async function includeEntities( folder: string, { type, clone, name, replace, }: { type: string; clone: boolean; name: string; replace: boolean }, ) { folder = path.resolve(folder); let variablePaths = globSync(path.join(folder, `**/variables.yaml`), { windowsPathsNoEscape: true, }) || []; if (!variablePaths || !variablePaths.length) { console.error( `Please choose a folder path that contains the variables.yaml file, e.g.: ~/workspaces/shopify/partials`, ); return; } folder = path.dirname(variablePaths[0]); let components: { filePaths: string[]; associations: | { taskId: string; pullerId: string; tableId: string; resultKey: string; } | { taskId: string; pullerId: string; tableId: string; resultKey: string; }[] | null; } | null = null; type = (type || '').trim().toLowerCase(); switch (type) { case 'all': components = await includeAll(folder); break; case 'task': components = await includeTask(folder, name); break; case 'connection': components = await includeConnection(folder, name); break; case 'pusher': components = await includePusher(folder, name); break; case 'dependency': components = await includeDependency(folder, name); break; case 'transformation': components = await includeTransformation(folder, name); break; case 'validation': components = await includeValidation(folder, name); break; case 'report': components = await includeReport(folder, name); break; case 'schema': components = await includeSchema(folder, name); break; default: components = await importCommon(folder, name); break; } if (!components || !components.filePaths || !components.filePaths.length) { return; } let variableKeys: string[] = []; for (var i = 0; i < components.filePaths.length; i++) { const filePath = components.filePaths[i]; const componentYml = fs.readFileSync(filePath, 'utf8'); variableKeys = [ ...variableKeys, ...(componentYml.match(/\*\*[^\*]+\*\*/gm) || []), ]; } variableKeys = variableKeys.filter((v, i, a) => a.indexOf(v) === i); const variabkeKeyNames = variableKeys.map((v, i) => `${i + 1}. ${v}`); let replacements: IExternalReplacement[] = [] as IExternalReplacement[]; if (!!replace && !clone) { console.log(`The included files contain the following variable keys: ${paddingList(variabkeKeyNames)} You can add replacements so that variable keys in the external files are mapped to your project's variable keys. This avoids importing variables that don't belong to your project. 1. Type the replacement in the format: '**SourceKey**' ==> '**TargetKey**' 2. Type 'done' when finished. 3. Type 'exit' or Ctrl+C to cancel.`); while (true) { const answer = await inquirer.prompt<{ value: string }>([ { type: 'input', name: 'value', message: `Replacement (e.g.: **MagentoConnector** ==> **MagentoConnectionId**):`, }, ]); const val = answer.value; if (!val) { console.log('Please input a valid message'); continue; } if (val.trim() === 'exit') { return; } if (val.trim() === 'done') { break; } const parts = val.split(/\s+==\>\s+/g); if (parts.length !== 2) { console.log(`Invalid format. Use: '**SourceKey**' ==> '**TargetKey**'`); continue; } replacements.push({ fromKey: parts[0].trim(), toKey: parts[1].trim(), value: null, }); console.log(`Added: ${parts[0].trim()} ==> ${parts[1].trim()}`); } } if (!!clone) { console.log(`There are several variables that need replacements as below: ${paddingList(variabkeKeyNames)} Please follow instructions to quickly replace the variable keys and values 1. Type the original text and the replacement text in the format of: 'original_key' ==> 'replacement_key' 2. Type 'done' to let the system process the replacement. 3. Type 'exit' or Ctrl+C to stop the replacement and quit if you think you are not correct.`); const globalReplacements: { fromText: string; toText: string }[] = []; while (true) { const answer = await inquirer.prompt<{ value: string }>([ { type: 'input', name: 'value', message: `Replacement (e.g.: Shopify_Product ==> Quickbooks_Item):`, }, ]); const val = answer.value; if (!val) { console.log('Please input a valid message'); continue; } if (val.trim() === 'exit') { return; } if (val.trim() === 'done') { break; } const replacement = val.split(/\s+==\>\s+/g); if (replacement.length !== 2) { console.log(`Please input a valid message, got: ${val}`); continue; } globalReplacements.push({ fromText: replacement[0], toText: replacement[1], }); } const variablesConf = parse(fs.readFileSync(variablePaths[0], 'utf8')); console.log(`Next we will ask you to override the values for the replacement keys. Please follow the instruction below: 1. Leave empty and Enter to keep the default value 2. Type uuid() to generate a new UUID string 3. Type any value to replace the default value`); for (var i = 0; i < variableKeys.length; i++) { const variableKey = variableKeys[i]; const rawKey = variableKey.replace(/\*\*/g, ''); let variableVal = variablesConf.variables[rawKey]; let replacementKey = variableKeys[i]; for (var j = 0; j < globalReplacements.length; j++) { replacementKey = replacementKey.replaceAll( globalReplacements[j].fromText, globalReplacements[j].toText, ); } const answer = await inquirer.prompt<{ value: string }>([ { type: 'input', name: 'value', message: `${replacementKey} (default '${variableVal}'):`, }, ]); let answerVal = answer.value; if ( !!answerVal && answerVal.replace(/[^a-zA-Z0-9\(\)]+/g, '')?.toLowerCase() === 'uuid()' ) { answerVal = v4(); } variableVal = answerVal || variableVal; if (variableKey !== replacementKey) { replacements.push({ fromKey: variableKey, toKey: replacementKey, value: variableVal, }); } } } const projectPath = getProjectPath(); // When using --replace (non-clone), skip transferring variables that have replacements // since they map to existing variables in the target project const replacedFromKeys = new Set(replacements.map((r) => r.fromKey)); const variableKeysToTransfer = variableKeys.filter( (k) => !replacedFromKeys.has(k), ); transferVariables(variableKeysToTransfer, folder, projectPath, replacements); for (var i = 0; i < components.filePaths.length; i++) { const filePath = components.filePaths[i]; includeYaml('local', filePath, replacements); } // Note: the task is complex because it must associate with a table, puller and pusher if (type === 'task') { const assocList = Array.isArray(components?.associations) ? (components?.associations as any[]) : components?.associations ? [components?.associations as any] : []; for (const assoc of assocList) { let associationYml = stringify(assoc || {}, { keepSourceTokens: true, lineWidth: 0, }); for (var i = 0; i < replacements.length; i++) { associationYml = associationYml.replace( replacements[i].fromKey, replacements[i].toKey, ); } const associations = parse(associationYml); const generatedAssociations = generateAssociation( associations.taskId!, { id: associations.tableId! }, { id: associations.pullerId!, resultKey: associations.resultKey!, }, ); fs.writeFileSync( generatedAssociations.filePath, stringify(generatedAssociations.config, { keepSourceTokens: true, lineWidth: 0, }), ); } } } export function IncludeCommand(): Command { const cmd = new Command('include') .alias('ic') .description('Include files from other project or components.') .argument('folder', 'Path to the target project') .option( '-t, --type ', 'Type of the entity to be included, e.g.: connection, task, puller, pusher, dependency, report, webhook, validation, transformation, mapping, ', ) .option('-c, --clone', 'Clone or duplicate the components') .option( '-r, --replace', 'Prompt to add variable key replacements for external files (e.g.: **MagentoConnector** ==> **MagentoConnectionId**)', ) .option( '-n, --name ', 'Search components by name, e.g.: product, sales order, customer', ) .action(includeEntities); return cmd; }