import { exec, spawn } from 'child_process'; import { privateDecrypt, privateEncrypt, RsaPrivateKey } from 'crypto'; import { readdir } from 'graceful-fs'; import { cloneDeep, isString } from 'lodash'; import { getType } from 'mime'; import { homedir, platform } from 'os'; import { join, normalize, parse } from 'path'; import { env } from 'process'; import { combineLatest, empty, merge, Observable, Observer, of, Subject } from 'rxjs'; import { catchError, filter, map, mergeMap, pluck, reduce, shareReplay, tap, toArray } from 'rxjs/operators'; import { VError } from 'verror'; import { Options } from './../options'; import { configTrimSlashes } from './../utils/config'; import { assertHasTrailingSlash, assertIsUrl, assertIsValidUserName, assertNotNull, isValidUrl, isValidUserName } from './assert'; import { canonicalizeJSON } from './json'; import { isJSON, readJSON } from './rx.json'; import { rxCacheSingle } from './rx.utils'; import { readFile, rxMkdirp, rxStats, walkFiles, writeFile as rxWriteFile } from './rx.walk'; const LAYOUT_MAPPINGS = 'layout-mappings'; const LAYOUTS = 'layouts'; const TYPES = 'types'; const SITES = 'sites'; const CONTENT = 'content'; const IMAGE_PROFILES = 'image-profiles'; const ASSETS = 'assets'; const RENDITIONS = 'renditions'; export interface Credentials { username: string; password: string; } export interface WchToolsOptions extends Credentials { baseUrl: string; } export interface IdMapping { [key: string]: T; } export interface SiteNode { children?: SiteNode[]; data?: any; } export interface WchTools { root: string; idToFile: IdMapping; layoutMappings: IdMapping; layouts: IdMapping; types: IdMapping; imageProfiles: IdMapping; content: IdMapping; sites: IdMapping; rxAssets: Observable>; rxRenditions: Observable>; } function _sortPages(aLeft: SiteNode, aRight: SiteNode): number { // check if we have data const posLeft = aLeft && aLeft.data && aLeft.data.position ? aLeft.data.position : 0, posRight = aRight && aRight.data && aRight.data.position ? aRight.data.position : 0; // compare return posLeft < posRight ? -1 : posLeft > posRight ? +1 : 0; } function _readSiteRecursive( aDir: string, aParent: SiteNode ): Observable { return Observable.create((observer: Observer) => { // check the type readdir(aDir, (err, files) => { // map from name to structure const children: { [name: string]: SiteNode } = {}; // handle the error if (err) { if (err.code === 'ENOENT') { observer.complete(); } else { observer.error(err); } } else { // for each file test if it's a file or const allFiles = files.map(name => { // get the status const fileName = join(aDir, name); return rxStats(fileName).pipe( mergeMap(stats => { if (stats.isDirectory()) { // make sure we have a node of that name const node = children[name] || (children[name] = {}); const ch = node.children || (node.children = []); // recurse return _readSiteRecursive(fileName, node); } else if ( stats.isFile() && getType(name) === 'application/json' ) { // split the file part const parsed = parse(name); const pageName = parsed.name; const node = children[pageName] || (children[pageName] = {}); // load the json return readFile(fileName).pipe( map(data => JSON.parse(data)), map(data => (node.data = data)), map(() => node) ); } else { return empty(); } }) ); }); // wait for all merge(...allFiles) .pipe( reduce((dst, src) => { return dst; }, children), map(data => { // validate the parent const ch = aParent.children || (aParent.children = []); // add all children // tslint:disable-next-line:forin for (const name in children) { ch.push(children[name]); } // order by position ch.sort(_sortPages); }), map(() => aParent) ) .subscribe(observer); } }); }); } function _readSite(aDir: string, aSite: SiteNode): Observable { // recurces return _readSiteRecursive(aDir, aSite); } function _isValidSite(aSite: any): boolean { return aSite && aSite.data && aSite.data.id; } function _readSites( aDir: string, aMappings: IdMapping ): Observable> { // construct a dummy helper const virtualRoot: SiteNode = {}; return _readSite(aDir, virtualRoot).pipe( map(root => { // reorder the sites into the mapping if (root.children) { root.children.filter(_isValidSite).forEach((site: any) => { aMappings[site.data.id] = site; }); } }), toArray(), map(() => aMappings) ); } function _readAll( aDir: string, aMappings: IdMapping, aNames: IdMapping ): Observable> { // iterate over all files return walkFiles(aDir).pipe( filter(isJSON), mergeMap(readJSON), reduce((dst, file) => { dst[file.data.id] = file.data; aNames[file.data.id] = file.desc.absPath; return dst; }, aMappings) ); } function _read(aDir: string): Observable { // read const files: IdMapping = {}; const layoutMappings = _readAll(join(aDir, LAYOUT_MAPPINGS), {}, files); const layouts = _readAll(join(aDir, LAYOUTS), {}, files); const types = _readAll(join(aDir, TYPES), {}, files); const sites = _readSites(join(aDir, SITES), {}); const content = _readAll(join(aDir, CONTENT), {}, files); const imageProfiles = _readAll(join(aDir, IMAGE_PROFILES), {}, files); // done return combineLatest( layoutMappings, layouts, types, sites, content, imageProfiles ).pipe( map(([lm, l, t, s, c, ip]) => { const tools = { root: aDir, layoutMappings: lm, layouts: l, types: t, idToFile: files, sites: s, content: c, imageProfiles: ip, rxAssets: rxCacheSingle(_readAll(join(aDir, ASSETS), {}, files)), rxRenditions: rxCacheSingle(_readAll(join(aDir, RENDITIONS), {}, files)) }; return tools; }) ); } /** * Makes sure our path ends with a proper trailing slash */ function _ensureTrailingSlash(aUrl: string): string { // test if the URL ends with a slash if (aUrl && aUrl.length > 0) { return aUrl[aUrl.length - 1] === '/' ? aUrl : aUrl + '/'; } // nothing to do return aUrl; } export const KEY_API_URL = 'x-ibm-dx-tenant-base-url'; export const KEY_USERNAME = 'username'; export const FILE_WCHTOOLS = '.wchtoolsoptions'; export const FILE_WCHTOOLS_JSON = '.wchtoolsoptions.json'; function _getApiBaseUrl( aOptions: Options, aDataDir?: string ): Observable { // verbose mode const bVerbose = !!aOptions.verbose; // check if we option const urlFromOptions = aOptions.url; if (urlFromOptions) { // some logging if (bVerbose) { console.log(`Using URL [${urlFromOptions}] from options.`); } return of(_ensureTrailingSlash(urlFromOptions)); } // check for the environment parameter const urlFromEnv = env[ENV_API_URL]; if (urlFromEnv) { // some logging if (bVerbose) { console.log(`Using URL [${urlFromEnv}] from environment.`); } return of(_ensureTrailingSlash(urlFromEnv)); } // data const dataDir = aDataDir || aOptions.data; // test for the existencee of a local file const localFile = join(dataDir || '.', FILE_WCHTOOLS), localJsonFile = join(dataDir || '.', FILE_WCHTOOLS_JSON), homeFile = join(homedir(), FILE_WCHTOOLS), homeJsonFile = join(homedir(), FILE_WCHTOOLS_JSON); // try to load the tools file return readFile(localFile).pipe( tap(() => { if (bVerbose) { console.log('read', localFile); } }), catchError(() => readFile(localJsonFile).pipe( tap(() => { if (bVerbose) { console.log('read', localJsonFile); } }) ) ), catchError(() => readFile(homeFile).pipe( tap(() => { if (bVerbose) { console.log('read', homeFile); } }) ) ), catchError(() => readFile(homeJsonFile).pipe( tap(() => { if (bVerbose) { console.log('read', homeJsonFile); } }) ) ), map(data => JSON.parse(data)), pluck(KEY_API_URL), filter(isValidUrl), map(_ensureTrailingSlash) ); } export declare type Matcher = (key: string) => boolean; function _getRegExp( aString: string | null | undefined, aDefault: boolean ): Matcher { if (aString) { // compile the pattern const pattern = new RegExp(aString); return (key: string) => pattern.test(key); } else { // always matches return (key: string) => aDefault; } } function _getPattern(aOptions: Options): Matcher { // get the pattern const include = _getRegExp(aOptions.include, true); const exclude = _getRegExp(aOptions.exclude, false); // return the combined function return (key: string) => include(key) && !exclude(key); } function _loadPrivateKey(): Observable { // filename const name = join(homedir(), '.ssh', 'id_rsa'); return readFile(name).pipe(map(key => ({ key }))); } const ENCRYPTED_ENCODING = 'base64'; const DECTYPTED_ENCODING = 'utf8'; function _encryptPassword(aPassword: string, aKey: RsaPrivateKey): string { // encrypt return privateEncrypt( aKey, Buffer.from(aPassword, DECTYPTED_ENCODING) ).toString(ENCRYPTED_ENCODING); } function _decryptPassword(aHash: string, aKey: RsaPrivateKey): string { return privateDecrypt(aKey, Buffer.from(aHash, ENCRYPTED_ENCODING)).toString( DECTYPTED_ENCODING ); } function _setCredentials( aWchToolsOptions: WchToolsOptions, aKey: RsaPrivateKey, aData: any ): any { // validate the URL assertHasTrailingSlash(aWchToolsOptions.baseUrl); // credentials const cred: Credentials = { username: aWchToolsOptions.username, password: _encryptPassword(aWchToolsOptions.password, aKey) }; // store aData[aWchToolsOptions.baseUrl] = cred; // ok return aData; } function _isValidCredential(aCred?: Credentials): boolean { return !!( aCred && isValidUserName(aCred.username) && isString(aCred.password) ); } function _emptyCredentials(): Credentials { return { username: '', password: '' }; } function _createCredentials(aUserName: string, aPassword: string): Credentials { return { username: aUserName, password: aPassword }; } const ENV_USERNAME = 'ibm_wch_sdk_cli_username'; const ENV_PASSWORD = 'ibm_wch_sdk_cli_password'; const ENV_API_URL = 'ibm_wch_sdk_cli_url'; function _getCredentialsFromEnvironment(): Credentials { // access the credentials from the environment const username = env[ENV_USERNAME] || ''; const password = env[ENV_PASSWORD] || ''; // construct return { username, password }; } /** * Merge different credentials layers * * @param aBase base layer * @param aOverride override layer * * @return the merged credentials */ function _mergeCredentials( aBase: Credentials, aOverride?: Credentials ): Credentials { // target if (!aOverride) { return aBase; } // clone const cred = cloneDeep(aBase); // override if (!!aOverride.username) { cred.username = aOverride.username; } if (!!aOverride.password) { cred.password = aOverride.password; } // ok return cred; } function _loadCredentials(aApiBase: string): Observable { // validate the URL assertHasTrailingSlash(aApiBase); // credential file name const filename = join(homedir(), '.ibm-wch-sdk-cli', '.credentials'); // read the credential const key = _loadPrivateKey(); // load the file return readFile(filename).pipe( map(data => JSON.parse(data)), map(data => data[aApiBase]), mergeMap(cred => key.pipe( map(k => _decryptPassword(cred.password, k)), map(p => { cred.password = p; return cred; }) ) ), catchError(() => of(_emptyCredentials())) ); } function _storeCredentials( aWchToolsOptions: WchToolsOptions ): Observable { // target folder const folder = join(homedir(), '.ibm-wch-sdk-cli'); const filename = join(folder, '.credentials'); // read the credential const key = _loadPrivateKey(); // create the directpry return rxMkdirp(folder).pipe( mergeMap(() => readFile(filename).pipe(map(data => JSON.parse(data)))), catchError(() => of({})), mergeMap(data => key.pipe(map(cred => _setCredentials(aWchToolsOptions, cred, data))) ), map(canonicalizeJSON), map(data => JSON.stringify(data)), mergeMap(data => rxWriteFile(filename, data)) ); } function _getWindowsCredentials(aApiUrl: string): Observable { // validate the URL assertHasTrailingSlash(aApiUrl); // the executable const path = normalize( join( __dirname, '..', '..', '..', 'assets', 'credman', process.arch, 'WchCredMan.exe' ) ); // execute const cmd = `\"${path}\" \"${aApiUrl}\"`; // construct the observable return Observable.create((observer: Observer) => { // execute the command exec( cmd, { encoding: 'utf8' }, (error, stdout, stderr) => { if (error) { observer.error(error); } else { try { // parse observer.next(JSON.parse(stdout)); observer.complete(); } catch (e) { observer.error(e); } } } ); }); } function _getStoredCredentials(aApiUrl: string): Observable { // the key const key = _ensureTrailingSlash(aApiUrl); // normalize the URL if (platform() === 'win32') { // load the credentials module return _getWindowsCredentials(key).pipe( mergeMap( (cred: Credentials) => _isValidCredential(cred) ? of(cred) : _loadCredentials(key) ), catchError(() => _loadCredentials(key)) ); } // linux like fallback return _loadCredentials(key); } function _getCredentials(aApiUrl: string): Observable { // return return _getStoredCredentials(aApiUrl).pipe( map(cred => _mergeCredentials(_getCredentialsFromEnvironment(), cred)), catchError(err => of(_getCredentialsFromEnvironment())) ); } function _getWchToolsOptions( aOptions: Options, aDataDir?: string ): Observable { // access the base URL return _getApiBaseUrl(aOptions, aDataDir).pipe( mergeMap(url => { // the result let result: Observable; // get the credentials const user = aOptions.user; const password = aOptions.password; if (user && password) { // ok result = of(_createCredentials(user, password)); } else { // read credentials from the store result = _getCredentials(url); } // build the options return result.pipe( map(cred => ({ baseUrl: _ensureTrailingSlash(url), username: cred.username, password: cred.password })) ); }), // make sure to share this shareReplay(1) ); } function _hidePassword(aArgs: string[]): string[] { // locate the password entry const copy = cloneDeep(aArgs); // find the pwd entry let idx = 0; while ((idx = copy.indexOf('--password', idx)) >= 0) { // replace next copy[++idx] = '***'; } return copy; } const noColors = /\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]/g; function _executeWchTools( aDataDir: string, aArgs: string[], aOptions: Options, aConfig: WchToolsOptions, aLogger: Subject ): Observable { // sanity checks assertNotNull(aDataDir, 'dataDir'); assertNotNull(aConfig, 'config'); assertIsValidUserName(aConfig.username, 'username'); assertNotNull(aConfig.password, 'password'); assertIsUrl(aConfig.baseUrl, 'apiUrl'); // check if this is a delete command const bDelete = aArgs && aArgs.length > 0 && aArgs[0] === 'delete'; // build the command line const args: string[] = [normalize(join(__dirname, 'fork.wchtools.js'))]; args.push( ...aArgs, '--user', aConfig.username, '--password', aConfig.password, '--url', aConfig.baseUrl ); // add the directory if (!bDelete) { args.push('--dir', aDataDir); } // verbose flag if (aOptions.verbose) { args.push('--verbose'); } // execute return Observable.create((o: Observer) => { // arguments const argString = JSON.stringify(_hidePassword(args)); // log this aLogger.next(`Spawning WCHTool with args [${argString}].`); // spawn the process const childProcess = spawn(process.execPath, args, { cwd: aDataDir, env: Object.assign( { NODE_TLS_REJECT_UNAUTHORIZED: '0' }, env ) }); // attach childProcess.stdout.on('data', data => aLogger.next(data.toString().replace(noColors, '')) ); childProcess.stderr.on('data', data => aLogger.next(data.toString().replace(noColors, '')) ); // end childProcess.on( 'close', code => code === 0 ? o.complete() : o.error( new VError( 'Exit code [%d] when executing wchtools with arguments [%s].', code, argString ) ) ); childProcess.on('error', err => o.error( new VError( err, 'Unable to execute wchtools with arguments [%s].', argString ) ) ); }); } function _getDeliveryUrlFromApiUrl(aApiUrl: string): string { // deduce the delivery URL from the api URL return aApiUrl.replace('/api/', '/'); } function _createUrl( aContextRoot: string, aOpts: WchToolsOptions ): Observable { // the delivery URL const absWchDeliveryUrl = _ensureTrailingSlash( _getDeliveryUrlFromApiUrl(aOpts.baseUrl) ); // path of the index file const absRootUrl = _ensureTrailingSlash( `${absWchDeliveryUrl}${configTrimSlashes(aContextRoot)}` ); // append the index const url = `${absRootUrl}index.html`; // ok return of(url); } export { _read as readWchTools, _getApiBaseUrl as wchToolsGetApiBaseUrl, _getPattern as wchToolsGetMatcher, _getCredentials as wchToolsGetCredentials, _getWchToolsOptions as wchToolsGetOptions, _executeWchTools as wchExecuteWchTools, _getDeliveryUrlFromApiUrl as wchDeliveryUrlFromApiUrl, _storeCredentials as wchStoreCredentials, _ensureTrailingSlash as ensureTrailingSlash, _createUrl as wchGetIndexUrl, _hidePassword as wchHidePassword };