import chalk from 'chalk'; import { IEntityColumn } from './@types/IEntityColumn'; import { IEntitySearchResult } from './@types/IEntitySearchResult'; import { findTableByEntityId, findTableById, selectTableColumnsVisibility, setTableColumnsIndexGroup, setTableColumnsPosition, } from './tableHelper'; import { getGlobalConfig, loadVariables } from './cluserHelper'; import { readVariableKey, readVariableValue } from './variablesHelper'; import { parse, stringify } from 'yaml'; import fs from 'fs'; import { Dictionary } from 'tsyringe/dist/typings/types'; import { IEntityInfo } from './@types/IEntityInfo'; import { entityFound } from './console'; import Fuse from 'fuse.js'; import inquirer from 'inquirer'; import plural from 'pluralize'; import fg from 'fast-glob'; import path from 'path'; import { selectTaskColumnsVisibility, setTaskColumnsConstraint, } from './taskHelper'; import { selectSchemaColumnsVisibility, setSchemaColumnsConstraint, } from './schemaHelper'; export const toFriendlyName = (originalName: string | null | undefined) => { if (!originalName) { return ''; } const blueParts = originalName.match(/\[([^\]]+)\]/g)?.map((part) => chalk.blue(part)) || []; // Extract and format remaining text const remainingText = originalName.replace(/\[([^\]]+)\]/g, '').trim(); const greenPart = chalk.green(remainingText); // Combine formatted parts const formattedName = [...blueParts, greenPart].join(' '); return formattedName; }; export function getCachePath( entityType: | 'connection' | 'task' | 'report' | 'schema' | 'puller' | 'pusher' | 'table' | 'task-relations', ): string { const cfg = getGlobalConfig(); switch (entityType) { case 'connection': return cfg.connectionCachesPath; case 'task': return cfg.taskCachesPath; case 'report': return cfg.reportCachesPath; case 'schema': return cfg.schemaCachesPath; case 'table': return cfg.tableCachesPath; case 'puller': return cfg.pullerCachesPath; case 'pusher': return cfg.pusherCachesPath; case 'task-relations': return cfg.taskRelationsCachesPath; default: console.error( `${chalk.red('✗')} Entity type ${chalk.red(entityType)} not supported.`, ); break; } return ''; } export function getEntityContentsProperty( entityType: 'connection' | 'task' | 'report' | 'schema' | 'table' | 'puller' | 'pusher', ): string { switch (entityType) { case 'connection': return 'connectors'; case 'task': return 'objects'; case 'report': return 'reports'; case 'schema': return 'schemas'; case 'table': return 'tables'; case 'puller': return 'pullers'; case 'pusher': return 'pushers'; default: console.error( `${chalk.red('✗')} Entity type ${chalk.red(entityType)} not supported.`, ); return ''; } } export async function loadCached( entityType: 'connection' | 'task' | 'report' | 'schema' | 'puller' | 'pusher' | 'table', ): Promise { const cachePath = getCachePath(entityType); const entityDisplayType = entityType .replace(/[^a-zA-Z0-9 ]/g, '') .replace(/\b\w/g, (l) => l.toUpperCase()); const cachedEntities = parse( fs.readFileSync(cachePath, 'utf8'), ) as Dictionary; if (!cachedEntities || !Object.keys(cachedEntities).length) { console.error( `${chalk.red('✗')} ${chalk.red(entityDisplayType)} caches not found in ${chalk.green(cachePath)}`, ); console.error( `${chalk.red('✗')} Please run ${chalk.green('hexasync profile set-context ')} to update the cache again.`, ); return []; } var variables = loadVariables(); var chosenEntities: Dictionary = {}; const prop = getEntityContentsProperty(entityType); for (var r of Object.values(cachedEntities).flat()) { const fileContent = parse(fs.readFileSync(r.filePath, 'utf8')) || {}; if (!fileContent || !fileContent[prop] || !fileContent[prop].length) { console.error( `${chalk.red('✗')} File content is empty or invalid for ${chalk.green(r.filePath)}.`, ); continue; } var entityContent = fileContent[prop].find( (e: any) => readVariableKey(e.id, '', variables) === readVariableKey(r.entityId, '', variables), ); if (!entityContent) { console.error( `${chalk.red('✗')} Entity content not found for ${chalk.green(r.entityId)} in file ${chalk.green(r.filePath)}.`, ); continue; } if (!r.importTypes.includes('override')) { chosenEntities[r.entityId] = { ...(chosenEntities[r.entityId] || {}), filePath: r.filePath as any, entityType: entityType, entityId: r.entityId, entityName: r.entityName, entityDescription: entityContent.entityDescription, content: entityContent, }; } else { chosenEntities[r.entityId] = { ...(chosenEntities[r.entityId] || {}), override: { filePath: r.filePath as any, content: entityContent }, }; } } var entityLists = Object.values(chosenEntities); return entityLists; } export async function findEntityById( entityId: string, entityType: 'task' | 'report' | 'schema' | 'table' | 'puller' | 'pusher', showsLogs: boolean = true, ): Promise { var variables = loadVariables(); var cachePath = getCachePath(entityType); let entityProp = getEntityContentsProperty(entityType); if (!cachePath || !entityProp) { console.error(`${chalk.red('✗')} Invalid cache path or entity property.`); return null; } const entityIdKey = readVariableKey(entityId, '', variables); const entityCached = parse(fs.readFileSync(cachePath, 'utf8')) as Dictionary< IEntityInfo[] >; const entityDisplayType = entityType .replace(/[^a-zA-Z0-9 ]/g, '') .replace(/\b\w/g, (l) => l.toUpperCase()); if (!entityCached) { console.error( `${chalk.red('✗')} ${chalk.red(entityDisplayType)} caches not found in ${chalk.red(cachePath)}`, ); console.error( `${chalk.red('✗')} Please run ${chalk.green('hexasync profile set-context ')} to update the cache again.`, ); return null; } if (!entityCached[entityIdKey] || !entityCached[entityIdKey].length) { console.error( `${chalk.red('✗')} ${chalk.red(entityDisplayType)} with key ${chalk.red(entityIdKey)} not found in cache.`, ); console.error( `${chalk.red('✗')} Please run ${chalk.green('hexasync profile set-context ')} to update the cache again.`, ); return null; } var foundEntity: IEntitySearchResult = null as any; for (const cachedEntity of entityCached[entityIdKey].sort( (a, b) => a.index - b.index, )) { const fileContent = parse(fs.readFileSync(cachedEntity.filePath, 'utf8')) || {}; if ( !fileContent || !fileContent[entityProp] || !fileContent[entityProp].length ) { console.error( `${chalk.red('✗')} ${chalk.red(entityDisplayType)} file ${chalk.green(cachedEntity.filePath)} is empty.`, ); continue; } const entityContent = fileContent[entityProp].find( (t) => (t.id = entityIdKey), ); if (!entityContent || !Object.keys(entityContent).length) { console.error( `${chalk.red('✗')} ${chalk.red(entityDisplayType)} with key ${chalk.green(entityIdKey)} not found in ${chalk.green(cachedEntity.filePath)}.`, ); continue; } if (!cachedEntity.importTypes.includes('override')) { if (showsLogs) entityFound( entityType, cachedEntity.entityId, cachedEntity.entityName, cachedEntity.filePath, ); foundEntity = { filePath: cachedEntity.filePath as any, entityType: entityType, entityId: entityContent.id, entityName: entityContent.name, entityDescription: entityContent.description, content: entityContent, }; } else { if (showsLogs) entityFound( `overrided ${entityType}`, cachedEntity.entityId, cachedEntity.entityName, cachedEntity.filePath, ); foundEntity = { ...foundEntity, override: { filePath: cachedEntity.filePath as any, content: entityContent, }, }; } } return foundEntity; } export async function searchEntityByName( name: string, entityType: 'task' | 'report' | 'schema' | 'puller' | 'pusher' | 'table', ): Promise { var cachePath = getCachePath(entityType); const cachedEntities = parse( fs.readFileSync(cachePath, 'utf8'), ) as Dictionary; const entityDisplayType = entityType .replace(/[^a-zA-Z0-9 ]/g, '') .replace(/\b\w/g, (l) => l.toUpperCase()); if (!cachedEntities || !Object.keys(cachedEntities).length) { console.error( `${chalk.red('✗')} ${chalk.red(entityDisplayType)} caches not found in ${chalk.green(cachePath)}`, ); console.error( `${chalk.red('✗')} Please run ${chalk.green('hexasync profile set-context ')} to update the cache again.`, ); return null; } const fuse = new Fuse(Object.values(cachedEntities).flat(), { includeScore: true, keys: ['entityName', 'filePath'], }); var searchedResults = fuse .search(name) .sort((a: any, b: any) => parseFloat(a.score) - parseFloat(b.score)) .map((r) => { return { ...r.item, score: r.score }; }); var chosenEntities: Dictionary = {}; var variables = loadVariables(); var prop = getEntityContentsProperty(entityType); for (var r of searchedResults) { var fileContent = parse(fs.readFileSync(r.filePath, 'utf8')); if (!fileContent || !fileContent[prop] || !fileContent[prop].length) { console.error( `${chalk.red('✗')} File content is empty or invalid for ${chalk.green(r.filePath)}.`, ); continue; } var entityContent = fileContent[prop].find( (e: any) => readVariableKey(e.id, '', variables) === readVariableKey(r.entityId, '', variables), ); if (!entityContent) { console.error( `${chalk.red('✗')} Entity content not found for ${chalk.green(r.entityId)} in file ${chalk.green(r.filePath)}.`, ); continue; } if (!r.importTypes.includes('override')) { chosenEntities[r.entityId] = { ...(chosenEntities[r.entityId] || {}), filePath: r.filePath as any, entityType: entityType, entityId: r.entityId, entityName: r.entityName, entityDescription: entityContent.entityDescription, content: entityContent, score: r.score, }; } else { chosenEntities[r.entityId] = { ...(chosenEntities[r.entityId] || {}), override: { filePath: r.filePath as any, content: entityContent }, }; } } var entityLists = Object.values(chosenEntities); var variables = loadVariables(); const cfg = getGlobalConfig(); if (entityLists.length > 1) { const answers = await inquirer.prompt([ { type: 'list', name: 'selectedEntity', message: chalk.gray( `Please choose the ${chalk.green(entityDisplayType.toUpperCase())} you would like to take actions (${chalk.white('ENTER')} to ${chalk.white('select')}, ${chalk.red('CTRL+C')} to ${chalk.red('exit')}):`, ), pageSize: 25, choices: entityLists.map((entity, index) => ({ name: `${index + 1}. ${toFriendlyName(readVariableValue(entity.entityName!, '', variables)?.toString())} (${chalk.gray(readVariableValue(entity.entityId, '', variables))}) - file: ${chalk.gray(entity.filePath.replace(cfg.componentPath, ''))}`, value: entity, })), }, ]); return answers.selectedEntity; } else if (entityLists.length === 1) { return entityLists[0]; } return null; } export async function searchEntitiesByName( name: string, entityType: 'task' | 'report' | 'schema' | 'table', ): Promise { var cachedPath = getCachePath(entityType); const entityDisplayType = entityType .replace(/[^a-zA-Z0-9 ]/g, '') .replace(/\b\w/g, (l) => l.toUpperCase()); const cachedEntities = parse( fs.readFileSync(cachedPath, 'utf8'), ) as Dictionary; if (!cachedEntities || !Object.keys(cachedEntities).length) { console.error( `${chalk.red('✗')} ${chalk.red(entityDisplayType)} caches not found in ${chalk.green(cachedPath)}`, ); console.error( `${chalk.red('✗')} Please run ${chalk.green('hexasync profile set-context ')} to update the cache again.`, ); return null; } const fuse = new Fuse(Object.values(cachedEntities).flat(), { includeScore: true, keys: ['entityName', 'filePath'], }); var searchedResults = fuse .search(name) .sort((a: any, b: any) => parseFloat(a.score) - parseFloat(b.score)) .map((r) => { return { ...r.item, score: r.score }; }); var chosenEntities: Dictionary = {}; var variables = loadVariables(); var prop = getEntityContentsProperty(entityType); for (var r of searchedResults) { var fileContent = parse(fs.readFileSync(r.filePath, 'utf8')); if (!fileContent || !fileContent[prop] || !fileContent[prop].length) { console.error( `${chalk.red('✗')} File content is empty or invalid for ${chalk.green(r.filePath)}.`, ); continue; } var entityContent = fileContent[prop].find( (e: any) => readVariableKey(e.id, '', variables) === readVariableKey(r.entityId, '', variables), ); if (!entityContent) { console.error( `${chalk.red('✗')} Entity content not found for ${chalk.green(r.entityId)} in file ${chalk.green(r.filePath)}.`, ); continue; } if (!r.importTypes.includes('override')) { chosenEntities[r.entityId] = { ...(chosenEntities[r.entityId] || {}), filePath: r.filePath as any, entityType: entityType, entityId: r.entityId, entityName: r.entityName, entityDescription: entityContent.entityDescription, content: entityContent, score: r.score, }; } else { chosenEntities[r.entityId] = { ...(chosenEntities[r.entityId] || {}), override: { filePath: r.filePath as any, content: entityContent }, }; } } var entityLists = Object.values(chosenEntities); var variables = loadVariables(); const entityDisplayTypePlural = plural(entityDisplayType); const cfg = getGlobalConfig(); if (entityLists.length > 1) { const answers = await inquirer.prompt([ { type: 'checkbox', name: 'selectedEntities', message: chalk.gray( `Please choose the ${chalk.green(entityDisplayTypePlural.toUpperCase())} you would like to take actions (${chalk.white('SPACE')} for ${chalk.white('select/unselect')}, ${chalk.green('ENTER')} to ${chalk.green('submit')}, ${chalk.red('CTRL+C')} to ${chalk.red('exit')}):`, ), pageSize: 25, choices: entityLists.map((entity, index) => ({ name: `${index + 1}. ${toFriendlyName(readVariableValue(entity.entityName!, '', variables)?.toString())} (${chalk.gray(readVariableValue(entity.entityId, '', variables))}) - file: ${chalk.gray(entity.filePath.replace(cfg.componentPath, ''))}`, value: entity, })), }, ]); return answers.selectedEntities; } if (entityLists.length === 1) { return entityLists; } return null; } export async function getEntityColumns( searchedEntity: IEntitySearchResult, ): Promise<{ table?: IEntitySearchResult | null; columns: IEntityColumn[]; } | null> { var allColumns: any = {}; var associatedTable: IEntitySearchResult | null = searchedEntity.entityType === 'table' ? searchedEntity : await findTableByEntityId( searchedEntity.entityId, searchedEntity.entityType as any, ); if (associatedTable) { if (!associatedTable?.content?.columns) { console.error( `${chalk.red('✗')} Columns not found for the associated table ${chalk.red(associatedTable?.entityId)}.`, ); return null; } allColumns = { ...associatedTable.content?.columns, ...associatedTable.override?.content?.columns, }; return { table: associatedTable, columns: Object.keys(allColumns).map((c) => { return { name: c, content: allColumns[c] }; }), }; } switch (searchedEntity.entityType) { case 'task': console.error( `${chalk.red('✗')} Falling back to find Task's Puller Mappings...`, ); var task = searchedEntity.content; if (!task?.puller?.mappings) { console.error(`${chalk.red('✗')} Task's Puller Mappings not found.`); return null; } allColumns = { ...task?.puller?.mappings, ...searchedEntity.override?.content?.puller?.mappings, }; break; case 'schema': var schema = searchedEntity.content; if (!schema?.columns?.length) { console.error( `${chalk.red('✗')} Columns not found for the schema ${chalk.red(searchedEntity.entityId)}.`, ); return null; } allColumns = { ...schema?.columns?.reduce((acc: any, c: any) => { acc[c.dbName] = c; return acc; }, {}), ...searchedEntity.override?.content?.columns?.reduce( (acc: any, c: any) => { acc[c.dbName] = c; return acc; }, {}, ), }; break; default: console.error( `${chalk.red('✗')} Entity type ${chalk.red(searchedEntity.entityType)} not supported.`, ); break; } return { table: associatedTable, columns: Object.keys(allColumns).map((c) => { return { name: c, content: allColumns[c] }; }), }; } export async function findEntityByIdOrName( id: string, name, entityType: 'task' | 'report' | 'schema' | 'table' | 'puller', ): Promise { var entity: any = null; if (!id && !name) { console.error(`${chalk.red('✗')} Either ID or name must be provided.`); return null; } const entityDisplayType = entityType .replace(/[^a-zA-Z0-9 ]/g, '') .replace(/\b\w/g, (l) => l.toUpperCase()); if (id) { console.log( `${chalk.green('✓')} Searching ${entityDisplayType} by id ${chalk.green(readVariableKey(id))} (${chalk.gray(readVariableValue(id))})...`, ); entity = await findEntityById(id, entityType); } else if (name) { console.log( `${chalk.green('✓')} Searching ${entityDisplayType} by name ${chalk.green(name)}...`, ); entity = await searchEntityByName(name, entityType); } if (!entity) { if (!!id) { console.error( `${chalk.red('✗')} ${entityDisplayType} with id ${chalk.green(id)} ${chalk.red('not found')}`, ); } else { console.error( `${chalk.red('✗')} ${entityDisplayType} with name ${chalk.green(name)} ${chalk.red('not found')}`, ); } } return entity; } export async function findEntitiesByIdOrName( id: string, name, entityType: 'task' | 'report' | 'schema' | 'table', ): Promise { var searchedEntities: IEntitySearchResult[] = []; const entityDisplayType = entityType .replace(/[^a-zA-Z0-9 ]/g, '') .replace(/\b\w/g, (l) => l.toUpperCase()); if (!id && !name) { return []; } if (id) { console.log( `${chalk.green('✓')} Searching ${entityDisplayType} by id ${chalk.green(readVariableKey(id))} (${chalk.gray(readVariableValue(id))})...`, ); var e = await findEntityById(id, entityType); if (e) { searchedEntities.push(e); } } else { console.log( `${chalk.green('✓')} Searching ${entityDisplayType} by name ${chalk.green(name)}...`, ); if (name === '*') { searchedEntities = [...((await loadCached(entityType)) || [])]; } else { var entities = await searchEntitiesByName(name, entityType); if (entities && entities.length > 0) { searchedEntities = [...searchedEntities, ...entities]; } } } return searchedEntities || []; } export function saveEntity( content: any, entityType: 'task' | 'report' | 'schema' | 'table' | 'puller' | 'pusher', filePath: string, ) { const entityDisplayType = entityType .replace(/[^a-zA-Z0-9 ]/g, '') .replace(/\b\w/g, (l) => l.toUpperCase()); const prop = getEntityContentsProperty(entityType); var entityName = content.name ? readVariableValue(content.name) : readVariableKey(content.id); console.log( `${chalk.green('✓')} Saving ${chalk.green(entityDisplayType)} ${chalk.yellow(entityName)} to ${chalk.yellow(filePath)}...`, ); var fileContent = fs.readFileSync(filePath, 'utf8'); var jsonObject = parse(fileContent); jsonObject[prop] = jsonObject[prop] || []; var taskIndex = jsonObject[prop].findIndex((t: any) => t.id === content.id); if (taskIndex > -1) { delete content.filePath; jsonObject[prop][taskIndex] = content; } fs.writeFileSync(filePath, stringify(jsonObject, { lineWidth: 0 })); console.log( `${chalk.green('✔')} ${entityDisplayType} ${chalk.yellow(entityName)} saved at ${chalk.yellow(filePath)}.`, ); } export async function cacheEntities( entityType: 'connection' | 'task' | 'puller' | 'pusher' | 'table' | 'schema' | 'report', ) { var variables = loadVariables(); var cfg = getGlobalConfig(); var cachePath = getCachePath(entityType); var prop = getEntityContentsProperty(entityType); var entities: Dictionary = {} as Dictionary; var mainYaml = parse(fs.readFileSync(cfg.mainFilePath, 'utf-8')); if (!mainYaml?.externals || !Array.isArray(mainYaml.externals)) { console.error( `${chalk.red('✗')} No external imports found in ${chalk.green(cfg.mainFilePath)}`, ); mainYaml.externals = []; } for (const external of mainYaml.externals) { var filePath = path.resolve(cfg.componentPath, external.path); if (!fs.existsSync(filePath)) { console.error( `${chalk.red('✗')} External file ${chalk.green(filePath)} not found`, ); return; } const fileContent = fs.readFileSync(filePath, 'utf8'); const jsonObject = parse(fileContent); if ( jsonObject[prop] && typeof jsonObject[prop] === 'object' && Object.keys(jsonObject[prop]).length > 0 ) { entities = { ...entities, ...jsonObject[prop].reduce((acc, o) => { var contentKey = readVariableKey(o.id, '', variables); var content = acc[contentKey] || []; acc[contentKey] = [ ...content, { filePath: filePath, index: external.index || 0, importTypes: ['external'], entityType: entityType, entityIdVariable: contentKey, entityId: readVariableValue(o.id, '', variables), entityNameVariable: !!o.name ? readVariableKey(o.name, '', variables) : '', entityName: !!o.name ? readVariableValue(o.name, '', variables) : '', }, ]; return acc; }, {}), }; } } for (const key in entities) { if (entities[key].length > 1) { console.error( `${chalk.red('✗')} ${entityType} with key ${chalk.green(key)} is ${chalk.red('included from external files more than once')}. `, ); for (var e of entities[key]) { console.log(` ${chalk.red(e.index)}. ${chalk.red(e.filePath)}`); } } } const stream = fg.stream( `${cfg.componentPath.replace(/\\/g, '/')}/**/*.yaml`, { onlyFiles: true, }, ); for await (const s of stream) { var filePath = s as string; const fileContent = fs.readFileSync(filePath, 'utf8'); const jsonObject = parse(fileContent); if ( jsonObject[prop] && typeof jsonObject[prop] === 'object' && Object.keys(jsonObject[prop]).length > 0 ) { for (var entity of jsonObject[prop]) { var contentKey = readVariableKey(entity.id, '', variables); var importTypes: ('external' | 'internal' | 'override')[] = entities[contentKey] && entities[contentKey].length ? ['internal', 'override'] : ['internal']; if (importTypes.includes('override')) { console.log( `${chalk.green('✓')} ${entityType} with key ${chalk.green(contentKey)} is included from ${chalk.green('external files and internal files')}. `, ); } entities[contentKey] = entities[contentKey] || []; entities[contentKey].push({ filePath: filePath, index: 99999999, importTypes: importTypes, entityType: entityType, entityIdVariable: contentKey, entityId: readVariableValue(entity.id, '', variables) as string, entityNameVariable: entity.name ? readVariableKey(entity.name, '', variables) : '', entityName: entity.name ? (readVariableValue(entity.name, '', variables) as string) : '', }); } } } console.log( `${chalk.green('✓')} Caching ${entityType}s to ${chalk.green(cachePath)}...`, ); fs.writeFileSync( cachePath, stringify(entities, { keepSourceTokens: true, lineWidth: 0 }), ); return entities; } export async function setEntityColumnsVisibility( id: string | undefined, name: string | undefined, entityType: 'task' | 'report' | 'schema' | 'table', ) { if (!id && !name) { console.error( `${chalk.red('✗')} Option ${chalk.green('id')} or ${chalk.green('name')} is ${chalk.red('required')}`, ); return; } var searchedTask = await findEntityByIdOrName(id!, name, entityType); if (!searchedTask) { return; } var taskColumns = (await getEntityColumns(searchedTask)) || { table: null, columns: [], }; if (!taskColumns?.columns?.length) { console.error( `${chalk.red('✗')} Columns not found for the ${chalk.red(entityType)} ${chalk.red(searchedTask?.entityId)}.`, ); return; } var answers = await inquirer.prompt([ { type: 'checkbox', name: 'selectedColumns', pageSize: 25, message: `Select columns to set ${chalk.yellow('VISIBLE')} (SPACE for select / unselect, ENTER for submit, CTRL+C to cancel):`, choices: taskColumns.columns.map((col, index) => ({ name: col.name, value: col.name, short: col.name, checked: col.content?.visibility === 1 || col.content?.visibility === 'Visible', })), }, ]); console.log('\n'); if (taskColumns.table) { var tableContent = selectTableColumnsVisibility( taskColumns.table, answers.selectedColumns, ); saveEntity(tableContent, 'table', tableContent.filePath); } else { switch (entityType) { case 'task': var taskContent = selectTaskColumnsVisibility(searchedTask, [ answers.selectedColumn, ]); saveEntity(taskContent, 'task', taskContent.filePath); break; case 'schema': var schemaContent = selectSchemaColumnsVisibility(searchedTask, [ answers.selectedColumn, ]); saveEntity(schemaContent, 'schema', schemaContent.filePath); break; default: console.error( `${chalk.red('✗')} Entity type ${chalk.red(entityType)} not supported.`, ); break; } } } export async function setEntityColumnsTrackForChanges( id: string | undefined, name: string | undefined, entityType: 'task' | 'report' | 'schema' | 'table', ) { if (!id && !name) { console.error( `${chalk.red('✗')} Option ${chalk.green('id')} or ${chalk.green('name')} is ${chalk.red('required')}`, ); return; } var searchedTask = await findEntityByIdOrName(id!, name, entityType); if (!searchedTask) { return; } var taskColumns = (await getEntityColumns(searchedTask)) || { table: null, columns: [], }; if (!taskColumns?.columns?.length) { console.error( `${chalk.red('✗')} Columns not found for the ${chalk.red(entityType)} ${chalk.red(searchedTask?.entityId)}.`, ); return; } var answers = await inquirer.prompt([ { type: 'checkbox', name: 'selectedColumns', pageSize: 25, message: `Select columns to set as ${chalk.yellow('TRACK_FOR_CHANGES')} (SPACE for select / unselect, ENTER for submit, CTRL+C to cancel):`, choices: taskColumns.columns.map((col, index) => ({ name: col.name, value: col.name, short: col.name, checked: col.content?.indexGroups?.includes('TRACK_FOR_CHANGES') || col.content?.constraints?.includes('VALUE_COMPARABLE'), })), }, ]); console.log('\n'); if (taskColumns.table) { var tableContent = setTableColumnsIndexGroup( taskColumns.table, answers.selectedColumns, 'TRACK_FOR_CHANGES', ); saveEntity(tableContent, 'table', tableContent.filePath); } else { switch (entityType) { case 'task': var taskContent = setTaskColumnsConstraint( searchedTask, [answers.selectedColumn], 'VALUE_COMPARABLE', ); saveEntity(taskContent, 'task', taskContent.filePath); break; case 'schema': var schemaContent = setSchemaColumnsConstraint( searchedTask, [answers.selectedColumn], 'VALUE_COMPARABLE', ); saveEntity(schemaContent, 'schema', schemaContent.filePath); break; default: console.error( `${chalk.red('✗')} Entity type ${chalk.red(entityType)} not supported.`, ); break; } } } export async function setEntityColumnsSourceId( id: string | undefined, name: string | undefined, entityType: 'task' | 'report' | 'schema' | 'table', ) { if (!id && !name) { console.error( `${chalk.red('✗')} Option ${chalk.green('id')} or ${chalk.green('name')} is ${chalk.red('required')}`, ); return; } var searchedTask = await findEntityByIdOrName(id!, name, entityType); if (!searchedTask) { return; } var taskColumns = (await getEntityColumns(searchedTask)) || { table: null, columns: [], }; if (!taskColumns?.columns?.length) { console.error( `${chalk.red('✗')} Columns not found for the ${chalk.red(entityType)} ${chalk.red(searchedTask?.entityId)}.`, ); return; } var answers = await inquirer.prompt([ { type: 'list', name: 'selectedColumn', pageSize: 25, message: `Select a column to set as ${chalk.yellow('SOURCE_ID')} (ENTER for submit, CTRL+C to cancel):`, choices: taskColumns.columns.map((col, index) => ({ name: col.name, value: col.name, short: col.name, checked: col.content?.indexGroups?.includes('SOURCE_ID') || col.content?.constraints?.includes('PRIMARY_KEY'), })), }, ]); console.log('\n'); if (taskColumns.table) { var tableContent = setTableColumnsIndexGroup( taskColumns.table, [answers.selectedColumn], 'SOURCE_ID', ); saveEntity(tableContent, 'table', tableContent.filePath); } else { switch (entityType) { case 'task': var taskContent = setTaskColumnsConstraint( searchedTask, [answers.selectedColumn], 'PRIMARY_KEY', ); saveEntity(taskContent, 'task', taskContent.filePath); break; case 'schema': var schemaContent = setSchemaColumnsConstraint( searchedTask, [answers.selectedColumn], 'PRIMARY_KEY', ); saveEntity(schemaContent, 'schema', schemaContent.filePath); break; default: console.error( `${chalk.red('✗')} Entity type ${chalk.red(entityType)} not supported.`, ); break; } } } export async function setEntityColumnsKeys( id: string | undefined, name: string | undefined, entityType: 'task' | 'report' | 'schema' | 'table', ) { if (!id && !name) { console.error( `${chalk.red('✗')} Option ${chalk.green('id')} or ${chalk.green('name')} is ${chalk.red('required')}`, ); return; } var searchedTask = await findEntityByIdOrName(id!, name, entityType); if (!searchedTask) { return; } var taskColumns = (await getEntityColumns(searchedTask)) || { table: null, columns: [], }; if (!taskColumns?.columns?.length) { console.error( `${chalk.red('✗')} Columns not found for the ${chalk.red(entityType)} ${chalk.red(searchedTask?.entityId)}.`, ); return; } var answers = await inquirer.prompt([ { type: 'checkbox', name: 'selectedColumns', pageSize: 25, message: `Select columns to set as ${chalk.yellow('KEYS')} (SPACE for select / unselect, ENTER for submit, CTRL+C to cancel):`, choices: taskColumns.columns.map((col, index) => ({ name: col.name, value: col.name, short: col.name, checked: col.content?.indexGroups?.includes('KEYS') || col.content?.constraints?.includes('KEY'), })), }, ]); console.log('\n'); if (taskColumns.table) { var tableContent = setTableColumnsIndexGroup( taskColumns.table, answers.selectedColumns, 'KEYS', ); saveEntity(tableContent, 'table', tableContent.filePath); } else { switch (entityType) { case 'task': var taskContent = setTaskColumnsConstraint( searchedTask, [answers.selectedColumn], 'KEY', ); saveEntity(taskContent, 'task', taskContent.filePath); break; case 'schema': var schemaContent = setSchemaColumnsConstraint( searchedTask, [answers.selectedColumn], 'KEY', ); saveEntity(schemaContent, 'schema', schemaContent.filePath); break; default: console.error( `${chalk.red('✗')} Entity type ${chalk.red(entityType)} not supported.`, ); break; } } } export function findLineNumber( filePath: string, searchText: string, ): number | null { try { const lines = fs.readFileSync(filePath, 'utf8').split('\n'); for (let i = 0; i < lines.length; i++) { if (lines[i].includes(searchText)) { return i + 1; } } } catch { // ignore } return null; } export function findEntityIndexLine( lines: string[], entityRawId: string, entityResolvedId: string, ): number | null { let entityLine = -1; for (let i = 0; i < lines.length; i++) { if (lines[i].includes(entityRawId) || lines[i].includes(entityResolvedId)) { entityLine = i; break; } } if (entityLine === -1) return null; for (let i = entityLine; i < Math.min(entityLine + 15, lines.length); i++) { if (/^\s*index\s*:/.test(lines[i])) { return i + 1; } } return null; } export interface IEntityIndexInfo { indexValue: string | number; /** Absolute path to the file where the index is defined. */ definedAt: string; /** 1-based line number of the index definition, or null if it could not be determined. */ lineNumber: number | null; } export function getEntityIndexInfo( mainRelation: { filePath: string; entityId: string; entityName?: string }, entityType: string, variables: any, ): IEntityIndexInfo | null { if (entityType !== 'transformations' && entityType !== 'validations') { return null; } if (!fs.existsSync(mainRelation.filePath)) return null; try { const fileContent = fs.readFileSync(mainRelation.filePath, 'utf8'); const lines = fileContent.split('\n'); const jsonObject = parse(fileContent); const entityDict = jsonObject[entityType]; if (!entityDict || typeof entityDict !== 'object') return null; for (const taskKey of Object.keys(entityDict)) { const entityList = entityDict[taskKey]; if (!Array.isArray(entityList)) continue; for (const entity of entityList) { const resolvedId = readVariableValue(entity.id, '', variables); if (resolvedId !== mainRelation.entityId) continue; if (entity.index === undefined || entity.index === null) return null; const rawIndex = entity.index; const isVariableRef = typeof rawIndex === 'string' && rawIndex.startsWith('**') && rawIndex.endsWith('**'); if (isVariableRef) { const indexValue = readVariableValue(rawIndex, '', variables); const variablesFilePath = path.resolve(variables.__file_path); const lineNumber = findLineNumber(variablesFilePath, rawIndex); return { indexValue: indexValue ?? rawIndex, definedAt: variablesFilePath, lineNumber, }; } else { const resolvedFilePath = path.resolve(mainRelation.filePath); const lineNumber = findEntityIndexLine( lines, entity.id, mainRelation.entityId, ); return { indexValue: rawIndex, definedAt: resolvedFilePath, lineNumber, }; } } } } catch { // ignore read errors } return null; } export async function setEntityColumnsPosition( id: string | undefined, name: string | undefined, entityType: 'task' | 'report' | 'schema' | 'table', ) { if (!id && !name) { console.error( `${chalk.red('✗')} Option ${chalk.green('id')} or ${chalk.green('name')} is ${chalk.red('required')}`, ); return; } var searchedEntity = await findEntityByIdOrName(id!, name, entityType); if (!searchedEntity) { return; } var entityColumns = (await getEntityColumns(searchedEntity)) || { table: null, columns: [], }; if (!entityColumns?.columns?.length) { console.error( `${chalk.red('✗')} Columns not found for the ${chalk.red(entityType)} ${chalk.red(searchedEntity?.entityId)}.`, ); return; } // Display current grid: column - visibility - position console.log(''); console.log( chalk.gray( ` ${'Column'.padEnd(40)} ${'Visibility'.padEnd(12)} ${'Position'}`, ), ); console.log( chalk.gray(` ${'─'.repeat(40)} ${'─'.repeat(12)} ${'─'.repeat(10)}`), ); for (const col of entityColumns.columns) { var vis = col.content?.visibility === 1 || col.content?.visibility === 'Visible' ? chalk.green('Visible') : chalk.gray('Hidden'); var pos = col.content?.position !== undefined ? chalk.yellow(String(col.content.position)) : chalk.gray('—'); console.log( ` ${chalk.white(col.name.padEnd(40))} ${vis.padEnd(21)} ${pos}`, ); } console.log(''); // Separate visible and hidden columns var visibleColumns = entityColumns.columns.filter( (col) => col.content?.visibility === 1 || col.content?.visibility === 'Visible', ); var hiddenColumns = entityColumns.columns.filter( (col) => col.content?.visibility !== 1 && col.content?.visibility !== 'Visible', ); if (visibleColumns.length === 0 && hiddenColumns.length === 0) { console.log(chalk.yellow('⚠︎ No columns found.')); return; } // Compute default positions for visible columns without a position // Priority: SOURCE_ID > KEYS > TRACK_FOR_CHANGES > alphabetical (A-Z) function getColumnSortPriority(col: IEntityColumn): number { var indexGroups: string[] = col.content?.indexGroups || []; if (indexGroups.includes('SOURCE_ID')) return 0; if (indexGroups.includes('KEYS')) return 1; if (indexGroups.includes('TRACK_FOR_CHANGES')) return 2; return 3; } var sortedVisibleColumns = [...visibleColumns].sort((a, b) => { var priorityA = getColumnSortPriority(a); var priorityB = getColumnSortPriority(b); if (priorityA !== priorityB) return priorityA - priorityB; return a.name.localeCompare(b.name); }); // Auto-assign default positions for visible columns that don't have one var visibleWithoutPosition = sortedVisibleColumns.filter( (col) => col.content?.position === undefined, ); if (visibleWithoutPosition.length > 0) { // Collect already-used positions from visible columns var existingPositions = new Set( sortedVisibleColumns .filter((col) => col.content?.position !== undefined) .map((col) => col.content.position as number), ); var nextPos = 0; var getNextFreePos = () => { while (existingPositions.has(nextPos)) nextPos++; return nextPos; }; console.log( chalk.gray( ' ⓘ Auto-assigning default positions for visible columns without position (SOURCE_ID > KEYS > TRACK_FOR_CHANGES > A-Z)', ), ); for (const col of visibleWithoutPosition) { var assignedPos = getNextFreePos(); col.content = col.content || {}; col.content.position = assignedPos; existingPositions.add(assignedPos); nextPos = assignedPos + 1; } // Re-display grid with defaults console.log(''); console.log( chalk.gray( ` ${'Column'.padEnd(40)} ${'Visibility'.padEnd(12)} ${'Position'}`, ), ); console.log( chalk.gray(` ${'─'.repeat(40)} ${'─'.repeat(12)} ${'─'.repeat(10)}`), ); for (const col of entityColumns.columns) { var vis = col.content?.visibility === 1 || col.content?.visibility === 'Visible' ? chalk.green('Visible') : chalk.gray('Hidden'); var pos = col.content?.position !== undefined ? chalk.yellow(String(col.content.position)) : chalk.gray('—'); console.log( ` ${chalk.white(col.name.padEnd(40))} ${vis.padEnd(21)} ${pos}`, ); } console.log(''); } // All visible columns are always selected (position is required for them) // User can optionally select hidden columns to set positions var selectedVisibleNames = sortedVisibleColumns.map((col) => col.name); if (hiddenColumns.length > 0) { var hiddenAnswers = await inquirer.prompt([ { type: 'checkbox', name: 'selectedHiddenColumns', pageSize: 25, message: `Optionally select ${chalk.gray('hidden')} columns to set ${chalk.yellow('POSITION')} (SPACE for select / unselect, ENTER to skip):`, choices: hiddenColumns.map((col) => ({ name: `${col.name} ${col.content?.position !== undefined ? chalk.gray(`(current: ${col.content.position})`) : ''}`, value: col.name, short: col.name, checked: col.content?.position !== undefined, })), }, ]); var selectedHiddenNames: string[] = hiddenAnswers.selectedHiddenColumns || []; var allSelectedNames = [...selectedVisibleNames, ...selectedHiddenNames]; var allSelectedColumns = [ ...sortedVisibleColumns, ...hiddenColumns.filter((c) => selectedHiddenNames.includes(c.name)), ]; // Ask position for each column, validating uniqueness var positions: Record = {}; var usedPositions: Set = new Set(); function getNextAvailablePosition(): number { var p = 0; while (usedPositions.has(p)) p++; return p; } for (const col of allSelectedColumns) { var defaultPos = col.content?.position !== undefined ? col.content.position : getNextAvailablePosition(); var { position } = await inquirer.prompt([ { type: 'input', name: 'position', message: `Position for ${chalk.green(col.name)}${col.content?.position !== undefined ? chalk.gray(` (current: ${col.content.position})`) : ''}:`, default: String(defaultPos), validate: (input: string) => { var num = parseInt(input, 10); if (isNaN(num) || num < 0) { return chalk.red('✗ Position must be a non-negative integer.'); } if (usedPositions.has(num)) { return chalk.red( `✗ Position ${num} is already used. Choose a different position.`, ); } return true; }, }, ]); var posNum = parseInt(position, 10); usedPositions.add(posNum); positions[col.name] = posNum; } // Build final positions: visible always have position, hidden only if selected var allPositions: Record = {}; for (const col of entityColumns.columns) { if (positions[col.name] !== undefined) { allPositions[col.name] = positions[col.name]; } else { allPositions[col.name] = undefined; } } // Duplicate check var positionValues = Object.values(allPositions).filter( (p) => p !== undefined, ) as number[]; var uniquePositions = new Set(positionValues); if (uniquePositions.size !== positionValues.length) { console.error( chalk.red('✗ Duplicate positions detected. Please try again.'), ); return; } console.log(''); if (entityColumns.table) { var tableContent = setTableColumnsPosition( entityColumns.table, allPositions, ); saveEntity(tableContent, 'table', tableContent.filePath); } else { console.error( `${chalk.red('✗')} Entity type ${chalk.red(entityType)} without a table is not supported for position setting.`, ); } // Show updated grid console.log(''); console.log(chalk.gray(' Updated positions:')); console.log( chalk.gray( ` ${'Column'.padEnd(40)} ${'Visibility'.padEnd(12)} ${'Position'}`, ), ); console.log( chalk.gray(` ${'─'.repeat(40)} ${'─'.repeat(12)} ${'─'.repeat(10)}`), ); for (const col of entityColumns.columns) { var vis = col.content?.visibility === 1 || col.content?.visibility === 'Visible' ? chalk.green('Visible') : chalk.gray('Hidden'); var pos = allPositions[col.name] !== undefined ? chalk.yellow(String(allPositions[col.name])) : chalk.gray('—'); console.log( ` ${chalk.white(col.name.padEnd(40))} ${vis.padEnd(21)} ${pos}`, ); } console.log(''); } else { // No hidden columns — just process visible columns var positions: Record = {}; var usedPositions: Set = new Set(); function getNextAvailablePosition(): number { var p = 0; while (usedPositions.has(p)) p++; return p; } for (const col of sortedVisibleColumns) { var defaultPos = col.content?.position !== undefined ? col.content.position : getNextAvailablePosition(); var { position } = await inquirer.prompt([ { type: 'input', name: 'position', message: `Position for ${chalk.green(col.name)}${col.content?.position !== undefined ? chalk.gray(` (current: ${col.content.position})`) : ''}:`, default: String(defaultPos), validate: (input: string) => { var num = parseInt(input, 10); if (isNaN(num) || num < 0) { return chalk.red('✗ Position must be a non-negative integer.'); } if (usedPositions.has(num)) { return chalk.red( `✗ Position ${num} is already used. Choose a different position.`, ); } return true; }, }, ]); var posNum = parseInt(position, 10); usedPositions.add(posNum); positions[col.name] = posNum; } var allPositions: Record = {}; for (const col of entityColumns.columns) { allPositions[col.name] = positions[col.name]; } console.log(''); if (entityColumns.table) { var tableContent = setTableColumnsPosition( entityColumns.table, allPositions, ); saveEntity(tableContent, 'table', tableContent.filePath); } else { console.error( `${chalk.red('✗')} Entity type ${chalk.red(entityType)} without a table is not supported for position setting.`, ); } // Show updated grid console.log(''); console.log(chalk.gray(' Updated positions:')); console.log( chalk.gray( ` ${'Column'.padEnd(40)} ${'Visibility'.padEnd(12)} ${'Position'}`, ), ); console.log( chalk.gray(` ${'─'.repeat(40)} ${'─'.repeat(12)} ${'─'.repeat(10)}`), ); for (const col of entityColumns.columns) { var vis = col.content?.visibility === 1 || col.content?.visibility === 'Visible' ? chalk.green('Visible') : chalk.gray('Hidden'); var pos = allPositions[col.name] !== undefined ? chalk.yellow(String(allPositions[col.name])) : chalk.gray('—'); console.log( ` ${chalk.white(col.name.padEnd(40))} ${vis.padEnd(21)} ${pos}`, ); } console.log(''); } }