"use strict";
import * as os from "os";
import * as fs from "fs";
import * as path from "path";
import * as util from "util";
import * as assert from "assert";
import * as glob from "glob";
import * as mkdirp from "mkdirp";
import * as detectIndent from "detect-indent";
// eslint-disable-next-line @typescript-eslint/no-var-requires
const pkg = require("../package");
const dtsExp = /\.d\.ts$/;
const bomOptExp = /^\uFEFF?/;
const externalExp = /^([ \t]*declare module )(['"])(.+?)(\2[ \t]*{?.*)$/;
const importExp = /^([ \t]*(?:)?(?:import .+? )= require\()(['"])(.+?)(\2\);.*)$/;
const importEs6Exp = /^([ \t]*(?:export|import) ?(?:(?:\* (?:as [^ ,]+)?)|.*)?,? ?(?:[^ ,]+ ?,?)(?:\{(?:[^ ,]+ ?,?)*\})? ?from )(['"])([^ ,]+)(\2;.*)$/;
const referenceTagExp = /^[ \t]*\/\/\/[ \t]*.*$/;
const identifierExp = /^\w+(?:[\.-]\w+)*$/;
const fileExp = /^([\./].*|.:.*)$/;
const privateExp = /^[ \t]*(?:static )?private (?:static )?/;
const publicExp = /^([ \t]*)(static |)(public |)(static |)(.*)/;
interface Options {
main: string;
name: string;
baseDir?: string;
out?: string;
newline?: string;
indent?: string;
outputAsModuleFolder?: boolean;
prefix?: string;
separator?: string;
externals?: boolean;
exclude?: { (file: string): boolean; } | RegExp;
removeSource?: boolean;
verbose?: boolean;
referenceExternals?: boolean;
emitOnIncludedFileNotFound?: boolean;
emitOnNoIncludedFileNotFound?: boolean;
headerPath: string;
headerText: string;
}
interface ModLine {
original: string;
modified?: string;
skip?: boolean;
}
interface Result {
file: string;
name: string;
indent: string;
exp: string;
refs: string[];
externalImports: string[];
relativeImports: string[];
exports: string[];
lines: ModLine[];
importLineRef: ModLine[];
relativeRef: ModLine[];
fileExists: boolean;
}
interface BundleResult {
fileMap: { [name: string]: Result; };
includeFilesNotFound: string[];
noIncludeFilesNotFound: string[];
emitted?: boolean;
options: Options;
}
export function bundle(options: Options): BundleResult {
assert(typeof options === "object" && options, "options must be an object");
// if main ends with **/*.d.ts all .d.ts files will be loaded
const allFiles = stringEndsWith(options.main, "**/*.d.ts");
// option parsing & validation
const main = allFiles ? "*.d.ts" : options.main;
const exportName = options.name;
const _baseDir = (() => {
let baseDir = optValue(options.baseDir, path.dirname(options.main));
if (allFiles) {
baseDir = baseDir!.substr(0, baseDir!.length - 2);
}
return baseDir;
})();
const out = optValue(options.out!, exportName + ".d.ts").replace(/\//g, path.sep);
const newline = optValue(options.newline, os.EOL);
const indent = optValue(options.indent, " ");
const outputAsModuleFolder = optValue(options.outputAsModuleFolder, false);
const prefix = optValue(options.prefix, "");
const separator = optValue(options.separator, "/");
const externals = optValue(options.externals, false);
const exclude = optValue(options.exclude, null);
const removeSource = optValue(options.removeSource, false);
const referenceExternals = optValue(options.referenceExternals, false);
const emitOnIncludedFileNotFound = optValue(options.emitOnIncludedFileNotFound, false);
const emitOnNoIncludedFileNotFound = optValue(options.emitOnNoIncludedFileNotFound, false);
const _headerPath = optValue(options.headerPath, null);
const headerText = optValue(options.headerText, "");
// regular (non-jsdoc) comments are not actually supported by declaration compiler
const comments = false;
const verbose = optValue(options.verbose, false);
assert.ok(main, 'option "main" must be defined');
assert.ok(exportName, 'option "name" must be defined');
assert(typeof newline === "string", 'option "newline" must be a string');
assert(typeof indent === "string", 'option "indent" must be a string');
assert(typeof prefix === "string", 'option "prefix" must be a string');
assert(separator!.length > 0, 'option "separator" must have non-zero length');
// turn relative paths into absolute paths
const baseDir = path.resolve(_baseDir!);
let mainFile = allFiles ? path.resolve(baseDir, "**/*.d.ts") : path.resolve(main.replace(/\//g, path.sep));
const outFile = calcOutFilePath(out, baseDir);
let headerData = "// Generated by dts-bundle v" + pkg.version + newline;
const headerPath = _headerPath && _headerPath !== "none" ? path.resolve(_headerPath.replace(/\//g, path.sep)) : _headerPath;
trace("### settings object passed ###");
traceObject(options);
trace("### settings ###");
trace("main: %s", main);
trace("name: %s", exportName);
trace("out: %s", out);
trace("baseDir: %s", baseDir);
trace("mainFile: %s", mainFile);
trace("outFile: %s", outFile);
trace("externals: %s", externals ? "yes" : "no");
trace("exclude: %s", exclude);
trace("removeSource: %s", removeSource ? "yes" : "no");
trace("comments: %s", comments ? "yes" : "no");
trace("emitOnIncludedFileNotFound: %s", emitOnIncludedFileNotFound ? "yes" : "no");
trace("emitOnNoIncludedFileNotFound: %s", emitOnNoIncludedFileNotFound ? "yes" : "no");
trace("headerPath %s", headerPath);
trace("headerText %s", headerText);
if (!allFiles) {
assert(fs.existsSync(mainFile), "main does not exist: " + mainFile);
}
if (headerPath) {
if (headerPath === "none") {
headerData = "";
} else {
assert(fs.existsSync(headerPath), "header does not exist: " + headerPath);
headerData = fs.readFileSync(headerPath, "utf8") + headerData;
}
} else if (headerText) {
headerData = "/*" + headerText + "*/\n";
}
let isExclude: (file: string, arg?: boolean) => boolean;
if (typeof exclude === "function") {
isExclude = exclude;
}
else if (exclude instanceof RegExp) {
isExclude = file => exclude.test(file);
}
else {
isExclude = () => false;
}
const sourceTypings = glob.sync("**/*.d.ts", { cwd: baseDir }).map(file => path.resolve(baseDir, file));
// if all files, generate temporally main file
if (allFiles) {
let mainFileContent = "";
trace("## temporally main file ##");
sourceTypings.forEach(file => {
const generatedLine = "* from './" + path.relative(baseDir, file.substr(0, file.length - 5)).replace(path.sep, "/") + "';";
trace(generatedLine);
mainFileContent += generatedLine + "\n";
});
mainFile = path.resolve(baseDir, "dts-bundle.tmp." + exportName + ".d.ts");
fs.writeFileSync(mainFile, mainFileContent, "utf8");
}
trace("\n### find typings ###");
const inSourceTypings = (file: string) => {
return sourceTypings.indexOf(file) !== -1 || sourceTypings.indexOf(path.join(file, "index.d.ts")) !== -1;
}; // if file reference is a directory assume commonjs index.d.ts
trace("source typings (will be included in output if actually used)");
sourceTypings.forEach(file => trace(" - %s ", file));
trace("excluded typings (will always be excluded from output)");
const fileMap: { [name: string]: Result; } = Object.create(null);
const globalExternalImports: string[] = [];
let mainParse: Result; // will be parsed result of first parsed file
const externalTypings: string[] = [];
const inExternalTypings = (file: string) => externalTypings.indexOf(file) !== -1;
{
// recursively parse files, starting from main file,
// following all references and imports
trace("\n### parse files ###");
const queue: string[] = [mainFile];
const queueSeen: { [name: string]: boolean; } = Object.create(null);
while (queue.length > 0) {
const target = queue.shift();
if (queueSeen[target!]) {
continue;
}
queueSeen[target!] = true;
// parse the file
const parse = parseFile(target!);
if (!mainParse!) {
mainParse = parse;
}
fileMap[parse.file] = parse;
pushUniqueArr(queue, parse.refs, parse.relativeImports);
}
}
// map all exports to their file
trace("\n### map exports ###");
const exportMap = Object.create(null);
Object.keys(fileMap).forEach(file => {
const parse = fileMap[file];
parse.exports.forEach(name => {
assert(!(name in exportMap), "already got for: " + name);
exportMap[name] = parse;
trace("- %s -> %s", name, parse.file);
});
});
// build list of typings to include in output later
trace("\n### determine typings to include ###");
const excludedTypings: string[] = [];
const usedTypings: Result[] = [];
const externalDependencies: string[] = []; // lists all source files that we omit due to !externals
{
const queue = [mainParse!];
const queueSeen: { [name: string]: boolean; } = Object.create(null);
trace("queue");
trace(queue);
while (queue.length > 0) {
const parse = queue.shift();
if (queueSeen[parse!.file]) {
continue;
}
queueSeen[parse!.file] = true;
trace("%s (%s)", parse!.name, parse!.file);
usedTypings.push(parse!);
parse!.externalImports.forEach(name => {
const p = exportMap[name];
if (!externals) {
trace(" - exclude external %s", name);
pushUnique(externalDependencies, !p ? name : p.file);
return;
}
if (isExclude(path.relative(baseDir, p.file), true)) {
trace(" - exclude external filter %s", name);
pushUnique(excludedTypings, p.file);
return;
}
trace(" - include external %s", name);
assert(p, name);
queue.push(p);
});
parse!.relativeImports.forEach(file => {
const p = fileMap[file];
if (isExclude(path.relative(baseDir, p.file), false)) {
trace(" - exclude internal filter %s", file);
pushUnique(excludedTypings, p.file);
return;
}
trace(" - import relative %s", file);
assert(p, file);
queue.push(p);
});
}
}
// rewrite global external modules to a unique name
trace("\n### rewrite global external modules ###");
usedTypings.forEach(parse => {
trace(parse.name);
parse.relativeRef.forEach((line, i) => {
line.modified = replaceExternal(line.original, getLibName);
trace(" - %s ==> %s", line.original, line.modified);
});
parse.importLineRef.forEach((line, i) => {
if (outputAsModuleFolder) {
trace(" - %s was skipped.", line.original);
line.skip = true;
return;
}
if (importExp.test(line.original)) {
line.modified = replaceImportExport(line.original, getLibName);
} else {
line.modified = replaceImportExportEs6(line.original, getLibName);
}
trace(" - %s ==> %s", line.original, line.modified);
});
});
// build collected content
trace("\n### build output ###");
let content = headerData;
if (externalDependencies.length > 0) {
content += "// Dependencies for this module:" + newline;
externalDependencies.forEach(file => {
if (referenceExternals) {
content += formatReference(path.relative(baseDir, file).replace(/\\/g, "/")) + newline;
}
else {
content += "// " + path.relative(baseDir, file).replace(/\\/g, "/") + newline;
}
});
}
if (globalExternalImports.length > 0) {
content += newline;
content += globalExternalImports.join(newline) + newline;
}
content += newline;
// content += header.stringify(header.importer.packageJSON(pkg)).join(lb) + lb;
// content += lb;
// add wrapped modules to output
content += usedTypings.filter((parse: Result) => {
// Eliminate all the skipped lines
parse.lines = parse.lines.filter((line: ModLine) => {
return (true !== line.skip);
});
// filters empty parse objects.
return (parse.lines.length > 0);
}).map((parse: Result) => {
if (inSourceTypings(parse.file)) {
return formatModule(parse.file, parse.lines.map(line => {
return getIndenter(parse.indent, indent)(line);
}));
}
else {
return parse.lines.map(line => {
return getIndenter(parse.indent, indent)(line);
}).join(newline) + newline;
}
}).join(newline) + newline;
// remove internal typings, except the 'regenerated' main typing
if (removeSource) {
trace("\n### remove source typings ###");
sourceTypings.forEach(p => {
// safety check, only delete .d.ts files, leave our outFile intact for now
if (p !== outFile && dtsExp.test(p) && fs.statSync(p).isFile()) {
trace(" - %s", p);
fs.unlinkSync(p);
}
});
}
const inUsed = (file: string): boolean => {
return usedTypings.filter(parse => parse.file === file).length !== 0;
};
const bundleResult: BundleResult = {
fileMap,
includeFilesNotFound: [],
noIncludeFilesNotFound: [],
options
};
trace("## files not found ##");
for (const p in fileMap) {
const parse = fileMap[p];
if (!parse.fileExists) {
if (inUsed(parse.file)) {
bundleResult.includeFilesNotFound.push(parse.file);
warning(" X Included file NOT FOUND %s ", parse.file);
} else {
bundleResult.noIncludeFilesNotFound.push(parse.file);
trace(" X Not used file not found %s", parse.file);
}
}
}
// write main file
trace("\n### write output ###");
// write only if there aren't not found files or there are and option "emit file not found" is true.
if ((bundleResult.includeFilesNotFound.length == 0
|| (bundleResult.includeFilesNotFound.length > 0 && emitOnIncludedFileNotFound))
&& (bundleResult.noIncludeFilesNotFound.length == 0
|| (bundleResult.noIncludeFilesNotFound.length > 0 && emitOnNoIncludedFileNotFound))) {
trace(outFile);
{
const outDir = path.dirname(outFile);
if (!fs.existsSync(outDir)) {
mkdirp.sync(outDir);
}
}
fs.writeFileSync(outFile, content, "utf8");
bundleResult.emitted = true;
} else {
warning(" XXX Not emit due to exist files not found.");
trace("See documentation for emitOnIncludedFileNotFound and emitOnNoIncludedFileNotFound options.");
bundleResult.emitted = false;
}
// print some debug info
if (verbose) {
trace("\n### statistics ###");
trace("used sourceTypings");
sourceTypings.forEach(p => {
if (inUsed(p)) {
trace(" - %s", p);
}
});
trace("unused sourceTypings");
sourceTypings.forEach(p => {
if (!inUsed(p)) {
trace(" - %s", p);
}
});
trace("excludedTypings");
excludedTypings.forEach(p => {
trace(" - %s", p);
});
trace("used external typings");
externalTypings.forEach(p => {
if (inUsed(p)) {
trace(" - %s", p);
}
});
trace("unused external typings");
externalTypings.forEach(p => {
if (!inUsed(p)) {
trace(" - %s", p);
}
});
trace("external dependencies");
externalDependencies.forEach(p => {
trace(" - %s", p);
});
}
trace("\n### done ###\n");
// remove temporally file.
if (allFiles) {
fs.unlinkSync(mainFile);
}
return bundleResult;
function stringEndsWith(str: string, suffix: string) {
return str.indexOf(suffix, str.length - suffix.length) !== -1;
}
function stringStartsWith(str: string, prefix: string) {
return str.slice(0, prefix.length) == prefix;
}
// Calculate out file path (see #26 https://github.com/TypeStrong/dts-bundle/issues/26)
function calcOutFilePath(out: any, baseDir: any) {
let result = path.resolve(baseDir, out);
// if path start with ~, out parameter is relative from current dir
if (stringStartsWith(out, "~" + path.sep)) {
result = path.resolve(".", out.substr(2));
}
return result;
}
function traceObject(obj: any) {
if (verbose) {
console.log(obj);
}
}
function trace(...args: any[]) {
if (verbose) {
console.log(util.format.apply(null, args));
}
}
function warning(...args: any[]) {
console.log(util.format.apply(null, args));
}
function getModName(file: string) {
return path.relative(baseDir, path.dirname(file) + path.sep + path.basename(file).replace(/\.d\.ts$/, ""));
}
function getExpName(file: string) {
if (file === mainFile) {
return exportName;
}
return getExpNameRaw(file);
}
function getExpNameRaw(file: string) {
return prefix + exportName + separator + cleanupName(getModName(file));
}
function getLibName(ref: string) {
return getExpNameRaw(mainFile) + separator + prefix + separator + ref;
}
function cleanupName(name: string) {
return name.replace(/\.\./g, "--").replace(/[\\\/]/g, separator!);
}
function mergeModulesLines(lines: any) {
const i = (outputAsModuleFolder ? "" : indent);
return (lines.length === 0 ? "" : i + lines.join(newline! + i)) + newline;
}
function formatModule(file: string, lines: string[]) {
let out = "";
if (outputAsModuleFolder) {
return mergeModulesLines(lines);
}
out += "declare module '" + getExpName(file) + "' {" + newline;
out += mergeModulesLines(lines);
out += "}" + newline;
return out;
}
// main info extractor
function parseFile(file: string): Result {
const name = getModName(file);
trace("%s (%s)", name, file);
const res: Result = {
file: file,
name: name,
indent: indent!,
exp: getExpName(file),
refs: [], // triple-slash references
externalImports: [], // import()'s like "events"
relativeImports: [], // import()'s like "./foo"
exports: [],
lines: [],
fileExists: true,
// the next two properties contain single-element arrays, which reference the same single-element in .lines,
// in order to be able to replace their contents later in the bundling process.
importLineRef: [],
relativeRef: []
};
if (!fs.existsSync(file)) {
trace(" X - File not found: %s", file);
res.fileExists = false;
return res;
}
if (fs.lstatSync(file).isDirectory()) { // if file is a directory then lets assume commonjs convention of an index file in the given folder
file = path.join(file, "index.d.ts");
}
const code = fs.readFileSync(file, "utf8").replace(bomOptExp, "").replace(/\s*$/, "");
res.indent = detectIndent(code).indent || indent!;
// buffer multi-line comments, handle JSDoc
let multiComment: string[] = [];
let queuedJSDoc: string[] | null;
let inBlockComment = false;
const popBlock = () => {
if (multiComment.length > 0) {
// jsdoc
if (/^[ \t]*\/\*\*/.test(multiComment[0])) {
// flush but hold
queuedJSDoc = multiComment;
}
else if (comments) {
// flush it
multiComment.forEach(line => res.lines.push({ original: line }));
}
multiComment = [];
}
inBlockComment = false;
};
const popJSDoc = () => {
if (queuedJSDoc) {
queuedJSDoc.forEach(line => {
// fix shabby TS JSDoc output
const match = line.match(/^([ \t]*)(\*.*)/);
if (match) {
res.lines.push({ original: match[1] + " " + match[2] });
}
else {
res.lines.push({ original: line });
}
});
queuedJSDoc = null;
}
};
code.split(/\r?\n/g).forEach((line: any) => {
let match: string[];
// block comment end
if (/^[((=====)(=*)) \t]*\*+\//.test(line)) {
multiComment.push(line);
popBlock();
return;
}
// block comment start
if (/^[ \t]*\/\*/.test(line)) {
multiComment.push(line);
inBlockComment = true;
// single line block comment
if (/\*+\/[ \t]*$/.test(line)) {
popBlock();
}
return;
}
if (inBlockComment) {
multiComment.push(line);
return;
}
// blankline
if (/^\s*$/.test(line)) {
res.lines.push({ original: "" });
return;
}
// reference tag
if (/^\/\/\//.test(line)) {
const ref = extractReference(line);
if (ref) {
const refPath = path.resolve(path.dirname(file), ref);
if (inSourceTypings(refPath)) {
trace(" - reference source typing %s (%s)", ref, refPath);
} else {
const relPath = path.relative(baseDir, refPath).replace(/\\/g, "/");
trace(" - reference external typing %s (%s) (relative: %s)", ref, refPath, relPath);
if (!inExternalTypings(refPath)) {
externalTypings.push(refPath);
}
}
pushUnique(res.refs, refPath);
return;
}
}
// line comments
if (/^\/\//.test(line)) {
if (comments) {
res.lines.push({ original: line });
}
return;
}
// private member
if (privateExp.test(line)) {
queuedJSDoc = null;
return;
}
popJSDoc();
// import() statement or es6 import
if ((line.indexOf("from") >= 0 && (match = line.match(importEs6Exp))) ||
(line.indexOf("require") >= 0 && (match = line.match(importExp)))) {
const [_, lead, quote, moduleName, trail] = match;
assert(moduleName);
const impPath = path.resolve(path.dirname(file), moduleName);
// filename (i.e. starts with a dot, slash or windows drive letter)
if (fileExp.test(moduleName)) {
// TODO: some module replacing is handled here, whereas the rest is
// done in the "rewrite global external modules" step. It may be
// more clear to do all of it in that step.
const modLine: ModLine = {
original: lead + quote + getExpName(impPath) + trail
};
res.lines.push(modLine);
let full = path.resolve(path.dirname(file), impPath);
// If full is not an existing file, then let's assume the extension .d.ts
if (!fs.existsSync(full) || fs.existsSync(full + ".d.ts")) {
full += ".d.ts";
}
trace(" - import relative %s (%s)", moduleName, full);
pushUnique(res.relativeImports, full);
res.importLineRef.push(modLine);
}
// identifier
else {
const modLine: ModLine = {
original: line
};
trace(" - import external %s", moduleName);
pushUnique(res.externalImports, moduleName);
if (externals) {
res.importLineRef.push(modLine);
}
if (!outputAsModuleFolder) {
res.lines.push(modLine);
} else {
pushUnique(globalExternalImports, line);
}
}
}
// declaring an external module
// this triggers when we're e.g. parsing external module declarations, such as node.d.ts
else if ((match = line.match(externalExp))) {
const [_, _declareModule, _lead, moduleName, _trail] = match;
assert(moduleName);
trace(" - declare %s", moduleName);
pushUnique(res.exports, moduleName);
const modLine: ModLine = {
original: line
};
res.relativeRef.push(modLine); // TODO
res.lines.push(modLine);
}
// clean regular lines
else {
// remove public keyword
if ((match = line.match(publicExp))) {
const [_, sp, static1, _pub, static2, ident] = match;
line = sp + static1 + static2 + ident;
}
if (inSourceTypings(file)) {
// for internal typings, remove the 'declare' keyword (but leave 'export' intact)
res.lines.push({ original: line.replace(/^()?declare /g, "$1") });
}
else {
res.lines.push({ original: line });
}
}
});
return res;
}
}
function pushUnique(arr: T[], value: T) {
if (arr.indexOf(value) < 0) {
arr.push(value);
}
return arr;
}
function pushUniqueArr(arr: T[], ...values: T[][]) {
values.forEach(vs => vs.forEach(v => pushUnique(arr, v)));
return arr;
}
function formatReference(file: string) {
return '/// ';
}
function extractReference(tag: string) {
const match = tag.match(referenceTagExp);
if (match) {
return match[2];
}
return null;
}
function replaceImportExport(line: string, replacer: (str: string) => string) {
const match = line.match(importExp);
if (match) {
assert(match[4]);
if (identifierExp.test(match[3])) {
return match[1] + match[2] + replacer(match[3]) + match[4];
}
}
return line;
}
function replaceImportExportEs6(line: string, replacer: (str: string) => string) {
if (line.indexOf("from") < 0) {
return line;
}
const match = line.match(importEs6Exp);
if (match) {
assert(match[4]);
if (identifierExp.test(match[3])) {
return match[1] + match[2] + replacer(match[3]) + match[4];
}
}
return line;
}
function replaceExternal(line: string, replacer: (str: string) => string) {
const match = line.match(externalExp);
if (match) {
const [_, declareModule, beforeIndent, moduleName, afterIdent] = match;
assert(afterIdent);
if (identifierExp.test(moduleName)) {
return declareModule + beforeIndent + replacer(moduleName) + afterIdent;
}
}
return line;
}
function getIndenter(actual: string, use: string): (line: ModLine) => string {
if (actual === use || !actual) {
return line => line.modified || line.original;
}
return line => (line.modified || line.original).replace(new RegExp("^" + actual + "+", "g"), match => match.split(actual).join(use));
}
function optValue(passed: T, def: T): T {
if (typeof passed === "undefined") {
return def;
}
return passed;
}