import chalk from 'chalk'; import { IEntitySearchResult } from './@types/IEntitySearchResult'; import { loadVariables, getGlobalConfig, getProjectPath } from './cluserHelper'; import { getCachePath, loadCached, searchEntityByName } from './entityHelper'; import path from 'path'; import { Dictionary } from 'tsyringe/dist/typings/types'; import { IEntityInfo } from './@types/IEntityInfo'; import { createVariable, readVariableKey, readVariableValue, } from './variablesHelper'; import fs from 'fs'; import { parse, stringify } from 'yaml'; import fg from 'fast-glob'; import inquirer from 'inquirer'; import { findConnectorById } from './connectorHelper'; import { getAssociation, normalizeName } from './profileGeneratorHelper'; import { v4 as uuid } from 'uuid'; import pluralize from 'pluralize'; import { resolvedConnectorIdOf } from '@beehexa/hexasync-template-model'; export function setTaskColumnsConstraint( task: IEntitySearchResult, columns: string[], constraint: 'KEY' | 'PRIMARY_KEY' | 'VALUE_COMPARABLE', ) { var taskContent = task.override?.content ? { ...task.override.content } : { ...task.content }; var hasChanged = false; for (var i = 0; i < Object.keys(taskContent.puller.mappings).length; i++) { var key = Object.keys(taskContent.puller.mappings)[i]; taskContent.puller.mappings[key].constraints = taskContent.puller.mappings[key].constraints || []; if (!columns.find((c) => c === key)) { if (taskContent.puller.mappings[key].constraints.includes(constraint)) { hasChanged = true; console.log( `${chalk.red('✗')} Removing constraint ${chalk.green(constraint)} from column ${chalk.green(key)}`, ); } taskContent.puller.mappings[key].constraints = taskContent.puller.mappings[key].constraints.filter( (g) => g !== constraint, ) || []; continue; } if (!taskContent.puller.mappings[key].constraints.includes(constraint)) { hasChanged = true; console.log( `${chalk.green('✓')} Adding constraint ${chalk.green(constraint)} to column ${chalk.green(key)}`, ); taskContent.puller.mappings[key].constraints.push(constraint); } } if (!hasChanged) { console.log(`${chalk.green('✓')} Nothing has changed...`); } return { ...taskContent, filePath: task.override?.content ? task.override.filePath : task.filePath, }; } export function selectTaskColumnsVisibility( task: IEntitySearchResult, columns: string[], ) { var taskContent = task.override?.content ? { ...task.override.content } : { ...task.content }; let hasChanged = false; for (var i = 0; i < Object.keys(taskContent.puller.mappings).length; i++) { var key = Object.keys(taskContent.puller.mappings)[i]; var visibility = columns.find((c) => c === key) ? 'Visible' : 0; if (visibility === 0 && taskContent.puller.mappings[key].visibility !== 0) { hasChanged = true; console.log(`${chalk.red('✗')} Hiding column ${chalk.green(key)}`); } else if ( visibility === 'Visible' && taskContent.puller.mappings[key].visibility === 0 ) { hasChanged = true; console.log(`${chalk.green('✓')} Showing column ${chalk.green(key)}`); } taskContent.puller.mappings[key].visibility = visibility; } if (!hasChanged) { console.log(`${chalk.green('✓')} Nothing has changed...`); } return { ...taskContent, filePath: task.override?.content ? task.override.filePath : task.filePath, }; } export async function cacheTaskRelations( relationType: 'dependencies' | 'transformations' | 'validations', ) { switch (relationType) { case 'dependencies': await cacheTaskRelationsList(relationType); break; case 'transformations': case 'validations': await cacheTaskRelationDictionary(relationType); break; } } function getTaskPropByRelationType(relationType: string) { switch (relationType) { case 'dependencies': return 'objectId'; } console.error( `${chalk.red('✗')} Invalid relation type ${chalk.green(relationType)}`, ); return ''; } async function cacheTaskRelationsList(relationType: 'dependencies') { var cfg = getGlobalConfig(); var variables = loadVariables(); var relations: Dictionary = {} as Dictionary; var relationCachePath = getCachePath('task-relations'); relations = (fs.existsSync(relationCachePath) && parse(fs.readFileSync(relationCachePath, 'utf-8'))) || {}; var prop = getTaskPropByRelationType(relationType); if (!prop) { return; } // 1. read from main.yaml's externals var mainYaml = parse(fs.readFileSync(cfg.mainFilePath, 'utf-8')); if (!mainYaml?.externals || !Array.isArray(mainYaml.externals)) { console.error( `${chalk.red('✗')} No external imports found in ${chalk.green(cfg.mainFilePath)}`, ); mainYaml.externals = []; } for (const external of mainYaml.externals) { var filePath = path.resolve(cfg.componentPath, external.path); if (!fs.existsSync(filePath)) { console.error( `${chalk.red('✗')} External file ${chalk.green(filePath)} not found`, ); return; } const fileContent = fs.readFileSync(filePath, 'utf8'); const jsonObject = parse(fileContent); if ( !jsonObject[relationType] || typeof jsonObject[relationType] !== 'object' || Object.keys(jsonObject[relationType]).length < 0 ) { continue; } relations = { ...relations, ...jsonObject[relationType].reduce((acc, o) => { var contentKey = readVariableKey(o[prop]); var content = acc[contentKey] || relations[contentKey] || []; acc[contentKey] = [ ...content, { filePath: filePath, index: external.index || 0, // external files will be composed last importTypes: ['external'], entityType: relationType, entityIdVariable: readVariableKey(o.id, '', variables), entityId: readVariableValue(o.id, '', variables), entityNameVariable: !!o.name ? readVariableKey(o.name, '', variables) : '', entityName: !!o.name ? readVariableValue(o.name, '', variables) : '', }, ]; return acc; }, {}), }; } // 1.1 Validate if there is any entities[key] that has more than 1 item var groupedRelations: Dictionary = {}; for (const key in relations) { for (const relation of relations[key]) { if (relation.entityType !== relationType) { continue; } if (!groupedRelations[relation.entityId]) { groupedRelations[relation.entityId] = []; } groupedRelations[relation.entityId].push(relation.filePath); } } for (var key in groupedRelations) { if (groupedRelations[key].length <= 1) { continue; } console.error( `${chalk.red('✗')} ${relationType} with key ${chalk.green(key)} is ${chalk.red('included from external files more than once')}. `, ); for (var filePath of groupedRelations[key]) { console.log(` - ${chalk.red(filePath)}`); } } // 2. read from files const stream = fg.stream( `${cfg.componentPath.replace(/\\/g, '/')}/**/*.yaml`, { onlyFiles: true, }, ); for await (const s of stream) { var filePath = s as string; const fileContent = fs.readFileSync(filePath, 'utf8'); const jsonObject = parse(fileContent); if ( !jsonObject[relationType] || typeof jsonObject[relationType] !== 'object' || jsonObject[relationType].length < 0 ) { continue; } for (var entity of jsonObject[relationType]) { var contentKey = readVariableKey(entity[prop], '', variables); // console.log('list', prop, contentKey) var importTypes: ('external' | 'internal' | 'override')[] = relations[contentKey] && relations[contentKey].length ? ['internal', 'override'] : ['internal']; relations[contentKey] = relations[contentKey] || []; if (importTypes.includes('override')) { console.log( `${chalk.green('✓')} ${relationType} with key ${chalk.green(contentKey)} is included from ${chalk.green('external files and internal files')}. `, ); } relations[contentKey].push({ filePath: filePath, index: 99999999, // internal files will be composed last importTypes: importTypes, entityType: relationType, entityIdVariable: readVariableKey(entity.id, '', variables) as string, entityId: readVariableValue(entity.id, '', variables) as string, entityNameVariable: entity.name ? readVariableKey(entity.name, '', variables) : '', entityName: entity.name ? (readVariableValue(entity.name, '', variables) as string) : '', }); } } // save the caches to the appropriate path console.log( `${chalk.green('✓')} Caching ${relationType} to ${chalk.green(relationCachePath)}...`, ); fs.writeFileSync( relationCachePath, stringify(relations, { keepSourceTokens: true, lineWidth: 0 }), ); } async function cacheTaskRelationDictionary( relationType: 'transformations' | 'validations', ) { var cfg = getGlobalConfig(); var variables = loadVariables(); var relations: Dictionary = {} as Dictionary; var relationCachePath = getCachePath('task-relations'); relations = (fs.existsSync(relationCachePath) && parse(fs.readFileSync(relationCachePath, 'utf-8'))) || {}; // 1. read from main.yaml's externals var mainYaml = parse(fs.readFileSync(cfg.mainFilePath, 'utf-8')); if (!mainYaml?.externals || !Array.isArray(mainYaml.externals)) { console.error( `${chalk.red('✗')} No external imports found in ${chalk.green(cfg.mainFilePath)}`, ); mainYaml.externals = []; } for (const external of mainYaml.externals) { var filePath = path.resolve(cfg.componentPath, external.path); if (!fs.existsSync(filePath)) { console.error( `${chalk.red('✗')} External file ${chalk.green(filePath)} not found`, ); return; } const fileContent = fs.readFileSync(filePath, 'utf8'); const jsonObject = parse(fileContent); if ( !jsonObject[relationType] || typeof jsonObject[relationType] !== 'object' || Object.keys(jsonObject[relationType]).length < 0 ) { continue; } relations = { ...relations, ...Object.keys(jsonObject[relationType]).reduce((acc, o) => { var contentKey = readVariableKey(o); var existingContent = acc[contentKey] || relations[contentKey] || []; // Preserve existing content acc[contentKey] = [ ...existingContent, ...jsonObject[relationType][o].map((x) => { return { filePath: filePath, index: external.index || 0, // external files will be composed last importTypes: ['external'], entityType: relationType, entityIdVariable: readVariableKey(x.id, '', variables), entityId: readVariableValue(x.id, '', variables), entityNameVariable: !!x.name ? readVariableKey(x.name, '', variables) : '', entityName: !!x.name ? readVariableValue(x.name, '', variables) : '', }; }), ]; return acc; }, {}), }; } // 1.1 Validate if there is any entities[key] that has more than 1 item var groupedRelations: Dictionary = {}; for (const key in relations) { for (const relation of relations[key]) { if (relation.entityType !== relationType) { continue; } if (!groupedRelations[relation.entityId]) { groupedRelations[relation.entityId] = []; } groupedRelations[relation.entityId].push(relation.filePath); } } for (var key in groupedRelations) { if (groupedRelations[key].length <= 1) { continue; } console.error( `${chalk.red('✗')} ${relationType} with key ${chalk.green(key)} is ${chalk.red('included from external files more than once')}. `, ); for (var filePath of groupedRelations[key]) { console.log(` - ${chalk.red(filePath)}`); } } // 2. read from files const stream = fg.stream( `${cfg.componentPath.replace(/\\/g, '/')}/**/*.yaml`, { onlyFiles: true, }, ); for await (const s of stream) { var filePath = s as string; const fileContent = fs.readFileSync(filePath, 'utf8'); const jsonObject = parse(fileContent); if ( !jsonObject[relationType] || typeof jsonObject[relationType] !== 'object' || Object.keys(jsonObject[relationType]).length < 0 ) { continue; } for (var entityId of Object.keys(jsonObject[relationType])) { var contentKey = readVariableKey(entityId, '', variables); var importTypes: ('external' | 'internal' | 'override')[] = relations[contentKey] && relations[contentKey].length ? ['internal', 'override'] : ['internal']; relations[contentKey] = relations[contentKey] || []; if (importTypes.includes('override')) { console.log( `${chalk.green('✓')} ${relationType} with key ${chalk.green(contentKey)} is included from ${chalk.green('external files and internal files')}. `, ); } relations[contentKey] = [ ...relations[contentKey], ...jsonObject[relationType][entityId].map((x) => { return { filePath: filePath, index: 99999999, // internal files will be composed last importTypes: importTypes, entityType: relationType, entityIdVariable: readVariableKey(x.id, '', variables), entityId: readVariableValue(x.id, '', variables) as string, entityNameVariable: x.name ? readVariableKey(x.name, '', variables) : '', entityName: x.name ? (readVariableValue(x.name, '', variables) as string) : '', }; }), ]; } } // save the caches to the appropriate path console.log( `${chalk.green('✓')} Caching ${relationType} to ${chalk.green(relationCachePath)}...`, ); fs.writeFileSync( relationCachePath, stringify(relations, { keepSourceTokens: true, lineWidth: 0 }), ); } export interface ITaskTemplate { id: string; name: string; description: string; priority: number; valueTableName: string; ignoreRemoval: boolean; ignoreChange: boolean; type: 'SYNC' | 'REFERENCE' | 'REPORT' | 'MAPPING'; } export function createSystemPrefix(pullConnection, pushConnection) { var prefix = ''; if (pullConnection && pushConnection) { prefix = `[${readVariableValue(pullConnection.entityName)} > ${readVariableValue(pushConnection.entityName)}]`; } else if (pullConnection) { prefix = `[${readVariableValue(pullConnection.entityName)}]`; } else if (pushConnection) { prefix = `[${readVariableValue(pushConnection.entityName)}]`; } return prefix; } export async function createTask( taskName: string, taskNamePrefix: string = '', taskType: ITaskTemplate['type'] = 'SYNC', ): Promise<{ filePath: string; content: ITaskTemplate; pullConnection: IEntitySearchResult | null; pushConnection: IEntitySearchResult | null; pullConnector: IEntityInfo | null; pushConnector: IEntityInfo | null; taskNamePrefix: string; } | null> { const entityType = 'task'; const entityTypeDisplayName = normalizeName(entityType); if (!taskName) { // 1. Use inquirer to prompt the user for the task name let { name } = await inquirer.prompt([ { type: 'input', name: 'name', message: `ⓘ Enter the ${entityTypeDisplayName.displayName} name:`, validate: (input) => input.trim() !== '' || chalk.red(`✗ ${entityTypeDisplayName.displayName} name is required.`), }, ]); if (!name) { console.error( chalk.red(`✗ ${entityTypeDisplayName.displayName} name is required.`), ); return null; } taskName = name; } taskName = pluralize(taskName); // 2. List all connections in project using fast-glob var connections = await loadCached('connection'); // Default prefix connections (not real connections, used for task name prefixes) var defaultPrefixConnections: (IEntitySearchResult | null)[] = [ null, { filePath: '', entityType: 'connection', entityId: '__prefix_audit__', entityName: 'Audit', content: {}, }, { filePath: '', entityType: 'connection', entityId: '__prefix_reference__', entityName: 'Reference', content: {}, }, { filePath: '', entityType: 'connection', entityId: '__prefix_hexasync__', entityName: 'HexaSync', content: {}, }, ]; var allConnectionChoices = [...defaultPrefixConnections, ...connections]; // 3. Prompt the user to select a connection var { pullConnection, pushConnection } = await inquirer.prompt([ { type: 'list', name: 'pullConnection', message: chalk.gray( `ⓘ Which ${chalk.white('data source')} that the task will ${chalk.green('pull from')}?:`, ), choices: allConnectionChoices.map((connection, i) => ({ name: `${i + 1}. ${!!connection?.entityName ? readVariableValue(connection?.entityName!) : 'None'}`, value: connection, })), }, { type: 'list', name: 'pushConnection', message: chalk.gray( `ⓘ Which ${chalk.white('data source')} that the task will ${chalk.green('push to')}?:`, ), choices: allConnectionChoices.map((connection, i) => ({ name: `${i + 1}. ${!!connection?.entityName ? readVariableValue(connection?.entityName!) : 'None'}`, value: connection, })), }, ]); if (!taskNamePrefix) { taskNamePrefix = createSystemPrefix(pullConnection, pushConnection); } var tableNamePrefix = ''; // FR-1 / AD-24: the resolved identity, not the legacy field. The falsy branch here is SILENT — it yields // null and the generated table name simply loses its system prefix — so a connectorId-only connection was // degrading the output with no error at all. const pullIdentity = resolvedConnectorIdOf(pullConnection?.content); const pushIdentity = resolvedConnectorIdOf(pushConnection?.content); var pullConnector = pullIdentity ? findConnectorById(pullIdentity) : null; var pushConnector = pushIdentity ? findConnectorById(pushIdentity) : null; if (pullConnector && pushConnector) { tableNamePrefix = `${pullConnector.slug}_${pushConnector.slug}`; } else if (pullConnector) { tableNamePrefix = `${pullConnector.slug}`; } else if (pushConnector) { tableNamePrefix = `${pushConnector.slug}`; } var normalizedTaskName = normalizeName(taskName); var normalizedTaskNamePrefix = normalizeName(taskNamePrefix); var baseNamePrefix = normalizedTaskNamePrefix.underscoreDisplayName ? `${normalizedTaskNamePrefix.underscoreDisplayName}__` : ''; var defaultTableName = `__hss_${tableNamePrefix}_${normalizedTaskName.underscoreSlug}`; // Prompt user for table name let { tableName } = await inquirer.prompt([ { type: 'input', name: 'tableName', message: `ⓘ Enter the table name (default: ${chalk.green(defaultTableName)}):`, default: defaultTableName, validate: (input) => { if (!input.startsWith('__hss')) { return chalk.red("✗ Table name must start with '__hss'."); } if (!/^[a-z0-9_]+$/.test(input)) { return chalk.red('✗ Table name must be in snake case.'); } return true; }, }, ]); var fileName = `${baseNamePrefix}${normalizedTaskName.underscoreDisplayName}_Task.yaml`; var filePath = path.join(getProjectPath(), 'objects', fileName); if (fs.existsSync(filePath)) { console.error(chalk.red(`✗ Task already exists at ${filePath}`)); return null; } var taskTemplate: ITaskTemplate = { id: createVariable( `${baseNamePrefix}${normalizedTaskName.underscoreDisplayName}_Task_Id`, uuid(), ), name: `${taskNamePrefix} ${normalizedTaskName.displayName}`, description: '', priority: 999999, valueTableName: createVariable( `${baseNamePrefix}${normalizedTaskName.underscoreDisplayName}_Task_Table_Name`, tableName, ), ignoreRemoval: taskType === 'SYNC', ignoreChange: false, type: taskType, }; // 4. Write the task to file fs.writeFileSync( filePath, stringify( { objects: [taskTemplate] }, { keepSourceTokens: true, lineWidth: 0 }, ), ); console.log(chalk.green(`✓ Task created at ${chalk.gray(filePath)}`)); return { filePath: path.join(getProjectPath(), 'objects', fileName), taskNamePrefix: taskNamePrefix, content: taskTemplate, pullConnection, pushConnection, pullConnector, pushConnector, }; } export function associateTask( taskId: string, entityType: 'table' | 'puller' | 'pusher', entityId: string, options?: Dictionary, ): void { taskId = readVariableKey(taskId); entityId = readVariableKey(entityId); var associations = getAssociation()?.objectAssociations || {}; var taskAssociations = associations[taskId] || {}; associations[taskId] = { ...taskAssociations, [entityType]: { id: entityId, }, }; var associationPath = path.join(getProjectPath(), 'ObjectAssociations.yaml'); fs.writeFileSync( associationPath, stringify( { objectAssociations: associations }, { keepSourceTokens: true, lineWidth: 0 }, ), ); console.log( `${chalk.green('✓')} Task ${chalk.gray(taskId)} associated to ${chalk.gray(entityType)} ${chalk.gray(entityId)}`, ); } export async function searchAndAssociateEntityToTask( taskId: string, entityType: 'table' | 'puller' | 'pusher', entitySearchName: string, options?: Dictionary, ): Promise { if (!entitySearchName) { let { searchedName } = await inquirer.prompt([ { type: 'input', name: 'searchedName', message: `ⓘ Please type the ${chalk.yellow('name')} of the ${entityType} that you would like to associate to this task:`, validate: (input) => input.trim() !== '' || `${chalk.red('✗')} Name is required.`, }, ]); if (!searchedName) { console.error(chalk.red('✗ Name is required.')); } entitySearchName = searchedName; } var searchedEntity = await searchEntityByName(entitySearchName, entityType); if (!searchedEntity) { console.error(chalk.red(`✗ ${entityType} ${entitySearchName} not found.`)); return null; } associateTask(taskId, entityType, searchedEntity.entityId, { resultKey: 'items', }); return searchedEntity; }