#!/usr/bin/env ts-node-script 'use strict'; import MySQLDatabaseDiscovery from "./MySQLDatabaseDiscovery"; import MongoDBDatabaseDiscovery from "./MongoDBDatabaseDiscovery"; import OracleDatabaseDiscovery from "./OracleDatabaseDiscovery"; import PostgresDatabaseDiscovery from "./PostgreSQLDatabaseDiscovery"; import MSSQLDatabaseDiscovery from "./MSSQLDatabaseDiscovery"; import DB2DatabaseDiscovery from "./DB2DatabaseDiscovery" import FileHelper from "./utils/FileHelper"; const metaModelBuilder = require('./MetaModelBuilder'); const fileStream = require('fs'); const path = require('path'); const multiHashing= require('multihashing-async'); const ab2str = require('arraybuffer-to-string'); const jsonMerger = require("json-merger"); const jp = require('jsonpath'); const { Sequelize } = require( 'sequelize' ); export default class Customer { //codec table refers to https://github.com/multiformats/multicodec/blob/master/table.csv hashCustomer = async (columnName:string, codec) => { let mh = await multiHashing(columnName, codec); return mh; } getCustomerIDwithHashkeyMySQL = async (MySQLConfiguration, databaseName, tableName, columnNames, codec) => { let mySQLDatabaseDiscovery = new MySQLDatabaseDiscovery(MySQLConfiguration); let extractedCustomers = await mySQLDatabaseDiscovery.extractCustomer(MySQLConfiguration,databaseName,tableName,columnNames); //console.log(extractedCustomers); //get the MySql schema //let extractedSchema = await metaModelBuilder.extractMySqlSchema(MySQLConfiguration,[databaseName]); //extractedSchema = JSON.parse(extractedSchema); let customerWithHashKeys = []; for(let extractedCust of extractedCustomers) { //convert object type to JSON type let extractedCustomer = JSON.parse(JSON.stringify(extractedCust)); let customerWithHashKey = {}; for(let column of columnNames) { let field = extractedCustomer[column]; customerWithHashKey[column] = field; /* if(MySQLConfiguration.HASH_COLUMN === column) { //hash according to the chosen column name let columnValue = extractedCustomer[column]; let hashed = await this.hashCustomer(columnValue.toString(), codec); //add the hashed column to the customer customerWithHashKey['hashed'] = ab2str(hashed,'base64'); } */ } customerWithHashKey['privacyHashed'] = ab2str(JSON.stringify(extractedCust),'base64'); customerWithHashKeys.push(customerWithHashKey); } return customerWithHashKeys; } getCustomerIDwithHashkeyMongoDB = async (MongoDBConfiguration, documentName, collectionName, columnNames, codec) => { let mongoDbDatabaseDiscovery = new MongoDBDatabaseDiscovery(MongoDBConfiguration); let extractedCustomers = await mongoDbDatabaseDiscovery.extractCustomer(MongoDBConfiguration,documentName,collectionName,columnNames); //console.log(extractedCustomers); //get the MongoDb schema //let extractedSchema = await metaModelBuilder.extractMongoSchema(MongoDBConfiguration,[documentName]); //extractedSchema = JSON.parse(extractedSchema); //console.log(extractedSchema); let customerWithHashKeys = []; for(let extractedCust of extractedCustomers) { //console.log(ab2str(JSON.stringify(extractedCust),'base64')) //convert object type to JSON type let extractedCustomer = JSON.parse(JSON.stringify(extractedCust)); let customerWithHashKey = {}; for(let column of columnNames) { let field = extractedCustomer[column]; customerWithHashKey[column] = field; /* //do the hash per column if(MongoDBConfiguration.HASH_COLUMN === column) { //hash according to the chosen column name let columnValue = extractedCustomer[column]; if(columnValue === null) break; let hashed = await this.hashCustomer(columnValue.toString(), codec); //add the hashed column to the customer customerWithHashKey['hashed'] = ab2str(hashed,'base64'); } */ //hash per object } customerWithHashKey['privacyHashed'] = ab2str(JSON.stringify(extractedCust),'base64'); customerWithHashKeys.push(customerWithHashKey); } return customerWithHashKeys; } getCustomerIDwithHashkeyOracle = async (OracleConfiguration, serviceName, tableName, columnNames, codec) => { let oracleDatabaseDiscovery = new OracleDatabaseDiscovery(OracleConfiguration); let extractedCustomers:any = []; extractedCustomers = await oracleDatabaseDiscovery.extractCustomer(OracleConfiguration,serviceName,tableName,columnNames); //get the Oracle schema //let owners = OracleConfiguration.OWNERS //let extractedSchema = await metaModelBuilder.extractOracleSchema(OracleConfiguration,[owners]); //extractedSchema = JSON.parse(extractedSchema); let customerWithHashKeys = []; for(let extractedCust of extractedCustomers) { //convert object type to JSON type let extractedCustomer = JSON.parse(JSON.stringify(extractedCust)); let customerWithHashKey = {}; for(let column of columnNames) { let field = extractedCustomer[column]; customerWithHashKey[column] = field; /* if(OracleConfiguration.HASH_COLUMN === column) { //hash according to the chosen column name let columnValue = extractedCustomer[column]; let hashed = await this.hashCustomer(columnValue.toString(), codec); //add the hashed column to the customer customerWithHashKey['hashed'] = ab2str(hashed,'base64'); } //customerWithHashKey['schema'] = extractedSchema['nodes']; */ } customerWithHashKey['privacyHashed'] = ab2str(JSON.stringify(extractedCust),'base64'); customerWithHashKeys.push(customerWithHashKey); } return customerWithHashKeys; } getCustomerIDwithHashkeyPostgres = async (PostgresConfiguration, codec) => { let postgresDatabaseDiscovery = new PostgresDatabaseDiscovery(PostgresConfiguration); let extractedCustomers:any = []; extractedCustomers = await postgresDatabaseDiscovery.extractCustomer(PostgresConfiguration); let customerWithHashKeys = []; for(let extractedCust of extractedCustomers) { //convert object type to JSON type let extractedCustomer = JSON.parse(JSON.stringify(extractedCust)); let customerWithHashKey = {}; for(let column of PostgresConfiguration.COLUMN) { let field = extractedCustomer[column]; customerWithHashKey[column] = field; } customerWithHashKey['privacyHashed'] = ab2str(JSON.stringify(extractedCust),'base64'); customerWithHashKeys.push(customerWithHashKey); } //console.log(customerWithHashKeys) return customerWithHashKeys; } getCustomerIDwithHashkeyDB2 = async (DB2Configuration, codec) => { let db2DatabaseDiscovery = new DB2DatabaseDiscovery(DB2Configuration); let extractedCustomers:any = []; extractedCustomers = await db2DatabaseDiscovery.extractCustomer(DB2Configuration); let customerWithHashKeys = []; for(let extractedCust of extractedCustomers) { //convert object type to JSON type let extractedCustomer = JSON.parse(JSON.stringify(extractedCust)); let customerWithHashKey = {}; for(let column of DB2Configuration.COLUMN) { let field = extractedCustomer[column]; customerWithHashKey[column] = field; } customerWithHashKey['privacyHashed'] = ab2str(JSON.stringify(extractedCust),'base64'); customerWithHashKeys.push(customerWithHashKey); } //console.log(customerWithHashKeys) return customerWithHashKeys; } getCustomerIDwithHashkeyMSSQL = async (MSSQLConfiguration, codec) => { console.log('getCustomerIDwithHashkeyMSSQL') let mssqlDatabaseDiscovery = new MSSQLDatabaseDiscovery(MSSQLConfiguration); let extractedCustomers:any = []; extractedCustomers = await mssqlDatabaseDiscovery.extractCustomer(MSSQLConfiguration); let customerWithHashKeys = []; for(let extractedCust of extractedCustomers) { //convert object type to JSON type let extractedCustomer = JSON.parse(JSON.stringify(extractedCust)); let customerWithHashKey = {}; for(let column of MSSQLConfiguration.COLUMN) { let field = extractedCustomer[column]; customerWithHashKey[column] = field; } customerWithHashKey['privacyHashed'] = ab2str(JSON.stringify(extractedCust),'base64'); customerWithHashKeys.push(customerWithHashKey); } //console.log(customerWithHashKeys) return customerWithHashKeys; } //For generating customer hashed id in mysql generateMySqlCustomerHash = async(MySQLConfiguration) => { let databaseName = MySQLConfiguration.DATABASE; let tableName = MySQLConfiguration.TABLE; let columnName = MySQLConfiguration.COLUMN; let codec = 'sha2-512'; let customer = new Customer(); let idwithhasedkey = await customer.getCustomerIDwithHashkeyMySQL(MySQLConfiguration, databaseName, tableName, columnName, codec); //console.log(idwithhasedkey); return idwithhasedkey } //For generating customer hashed id in mongodb generateMongoDbCustomerHash = async(MongoDBConfiguration) => { let documentName = MongoDBConfiguration.DOCUMENT; let collectionName = MongoDBConfiguration.COLLECTION; let columnName = MongoDBConfiguration.COLUMN; let codec = 'sha2-512'; let customer = new Customer(); let idwithHashedKey = await customer.getCustomerIDwithHashkeyMongoDB(MongoDBConfiguration, documentName, collectionName, columnName, codec); //console.log(idwithHashedKey); return idwithHashedKey; } //For generating customer hashed id in Oracle generateOracleCustomerHash = async(OracleConfiguration) => { let serviceName = OracleConfiguration.SERVICE_NAME; let tableName = OracleConfiguration.TABLE; let columnName = OracleConfiguration.COLUMN; let codec = 'sha2-512'; let customer = new Customer(); let idwithhasedkey = await customer .getCustomerIDwithHashkeyOracle(OracleConfiguration, serviceName, tableName, columnName, codec); //console.log(idwithhasedkey); return idwithhasedkey; } generatePostgresCustomerHash = async(PostgresConfiguration) => { let codec = 'sha2-512'; let customer = new Customer(); let idwithhasedkey = await customer .getCustomerIDwithHashkeyPostgres(PostgresConfiguration, codec); //console.log(idwithhasedkey); return idwithhasedkey; } generateMSSQLCustomerHash = async(MSSQLConfiguration) => { let codec = 'sha2-512'; let customer = new Customer(); let idwithhasedkey = await customer .getCustomerIDwithHashkeyMSSQL(MSSQLConfiguration, codec); //console.log(idwithhasedkey); return idwithhasedkey; } generateDB2CustomerHash = async(DB2Configuration) => { let codec = 'sha2-512'; let customer = new Customer(); let idwithhasedkey = await customer .getCustomerIDwithHashkeyDB2(DB2Configuration, codec); //console.log(idwithhasedkey); return idwithhasedkey; } //Generate a customer hash based on (multiple) DBMS generateCustomerHash = async(configurationJson, dirName) => { const fileHelper = new FileHelper(); let customerHashFileNames = []; let customerHashFileName; let fileNamePrefix = "customer_hashed_"; let customerHash={}; let customerHashes=[]; let connections = JSON.parse(fileHelper.readFile(configurationJson)) const CUSTOMER_FCN = { MONGODB : this.generateMongoDbCustomerHash, MYSQL : this.generateMySqlCustomerHash, ORACLE : this.generateOracleCustomerHash, POSTGRES: this.generatePostgresCustomerHash, MSSQL : this.generateMSSQLCustomerHash, DB2 : this.generateDB2CustomerHash } let i=0; for(let connection of connections.configuration){ console.log(connection); customerHashFileName = `${fileNamePrefix}${i}.json` const DBMS = connection['DBMS'].toUpperCase(); //const databaseNames = connection['DATABASE'] ? connection['DATABASE'] : connection['OWNERS'] ; try { let customerData = await CUSTOMER_FCN[DBMS](connection); //console.log(customerData) //assign key with hashedId //customerHash['hashedId'] = jp.query(hashData, '$..hashedId') //console.log(customerHash); //customerHashFileNames.push(path.join(dirName, customerHashFileName)); //fileHelper.writeFile(path.join(dirName, customerHashFileName), JSON.stringify(customerData,null,4)); i++; //combine every customerHash file from every connection string into one customerHash file customerHashes = customerHashes.concat(customerData); //customerHashes = await jsonMerger.object([customerHashes,customerData]); //jsonMerger.mergeObjects([edges['edges'], reversedEdges], {defaultArrayMergeOperation: "concat"}) } catch (err) { console.log(err); } } //combine every schema file from every connection string into one schema file //customerHashes = await jsonMerger.mergeFiles(customerHashFileNames, {defaultArrayMergeOperation: "concat"}); return customerHashes; } harvestCustomerHash = (configFile, dirName, customerHashFilename) => { const fileHelper = new FileHelper(); let connections = fileHelper.readFile(configFile); //console.log(connections); if(!fileHelper.isJsonString(connections)) { console.log("Config file has an invalid Json String"); return; } //console.log(JSON.parse(connections['configuration'])); connections = JSON.parse(connections); /* //check for the configuration for hashed keys for(let configuration of connections['configuration']) { //check whether the hash field exists if(!fileHelper.isHashedFieldExist(configuration.HASH_COLUMN, configuration.COLUMN) ) { console.log(`Hash column : ${configuration.HASH_COLUMN} is not valid`); return; } } */ if (!fileStream.existsSync(dirName)) { console.log("Directory does not exist"); return; } //starts creating customerHash files and combine into one customerHash (async () => { console.log('Start creating a customerHash file') let customerHashedId = await this.generateCustomerHash(configFile,dirName); //console.log(customerHashedId); fileHelper.writeFile(path.join(dirName, customerHashFilename ), JSON.stringify(customerHashedId,null,4)); console.log(`${customerHashFilename} has been created`); })(); } } /******************** Main Entry Point ******************************************/ const customer = new Customer(); if(!process.argv[2] || !process.argv[3] || !process.argv[4]) { //npm run penta-customer-hash customer_config.json schemas customer_hashed10_all.json console.log('Please supply three arguments: configFile, outputDirectory and the outputFile'); } //else { // customer.harvestCustomerHash(process.argv[2], process.argv[3], process.argv[4]); //} else if( process.argv.length == 5 && process.argv[1].includes('Customer.ts')) { //npm run penta-customer-hashing customer_config.json schemas customer_hashed1_all.json customer.harvestCustomerHash(process.argv[2], process.argv[3], process.argv[4]); } else if( process.argv.length == 5 && process.argv[1].includes('customer-hash')) { //npx penta-customer-hash customer_config.json schemas customer_hashed1_all.json customer.harvestCustomerHash(process.argv[2], process.argv[3], process.argv[4]); }