import type { Blob, FilePath, BundleResult, Bundle as BundleType, BundleGraph as BundleGraphType, NamedBundle as NamedBundleType, Async, } from '@atlaspack/types'; import type SourceMap from '@atlaspack/source-map'; import type { Bundle as InternalBundle, Config, DevDepRequest, AtlaspackOptions, ReportFn, RequestInvalidation, DevDepRequestRef, } from './types'; import type {AtlaspackConfig, LoadedPlugin} from './AtlaspackConfig'; import type InternalBundleGraph from './BundleGraph'; import type {ConfigRequest} from './requests/ConfigRequest'; import type {DevDepSpecifier} from './requests/DevDepRequest'; import invariant from 'assert'; import {createBuildCache} from '@atlaspack/build-cache'; import {blobToStream, debugTools, TapStream} from '@atlaspack/utils'; import {PluginLogger} from '@atlaspack/logger'; import ThrowableDiagnostic, {errorToDiagnostic} from '@atlaspack/diagnostic'; import {Readable} from 'stream'; import nullthrows from 'nullthrows'; import path from 'path'; import {hashString, hashBuffer, Hash} from '@atlaspack/rust'; import {NamedBundle, bundleToInternalBundle} from './public/Bundle'; import BundleGraph, { bundleGraphToInternalBundleGraph, } from './public/BundleGraph'; import PluginOptions from './public/PluginOptions'; import PublicConfig from './public/Config'; import {ATLASPACK_VERSION, HASH_REF_PREFIX, HASH_REF_REGEX} from './constants'; import { fromProjectPath, toProjectPathUnsafe, fromProjectPathRelative, joinProjectPath, } from './projectPath'; import {createConfig} from './InternalConfig'; import { loadPluginConfig, getConfigHash, getConfigRequests, PluginWithBundleConfig, } from './requests/ConfigRequest'; import { createDevDependency, getWorkerDevDepRequests, } from './requests/DevDepRequest'; import {getInvalidationId, getInvalidationHash} from './assetUtils'; import {optionsProxy} from './utils'; import {invalidateDevDeps} from './requests/DevDepRequest'; import {computeSourceMapRoot} from './requests/WriteBundleRequest'; import {tracer, PluginTracer} from '@atlaspack/profiler'; import {fromEnvironmentId} from './EnvironmentManager'; type Opts = { config: AtlaspackConfig; options: AtlaspackOptions; report: ReportFn; previousDevDeps: Map; previousInvalidations: Array; }; export type RunPackagerRunnerResult = { bundleInfo: BundleInfo; configRequests: Array; devDepRequests: Array; invalidations: Array; }; export type BundleInfo = { readonly type: string; readonly size: number; readonly hash: string; readonly hashReferences: Array; readonly time?: number; readonly cacheKeys: CacheKeyMap; readonly isLargeBlob: boolean; readonly scopeHoistingStats?: { totalAssets: number; wrappedAssets: number; }; }; type CacheKeyMap = { content: string; map: string; info: string; }; const BOUNDARY_LENGTH = HASH_REF_PREFIX.length + 32 - 1; // Packager/optimizer configs are not bundle-specific, so we only need to // load them once per build. const pluginConfigs = createBuildCache(); export default class PackagerRunner { config: AtlaspackConfig; options: AtlaspackOptions; pluginOptions: PluginOptions; // @ts-expect-error TS2564 distDir: FilePath; // @ts-expect-error TS2564 distExists: Set; report: ReportFn; previousDevDeps: Map; devDepRequests: Map; invalidations: Map; previousInvalidations: Array; constructor({ config, options, report, previousDevDeps, previousInvalidations, }: Opts) { this.config = config; this.options = options; this.report = report; this.previousDevDeps = previousDevDeps; this.devDepRequests = new Map(); this.previousInvalidations = previousInvalidations; this.invalidations = new Map(); this.pluginOptions = new PluginOptions( optionsProxy(this.options, (option) => { let invalidation: RequestInvalidation = { type: 'option', key: option, }; this.invalidations.set(getInvalidationId(invalidation), invalidation); }), ); } async run( bundleGraph: InternalBundleGraph, bundle: InternalBundle, invalidDevDeps: Array, ): Promise { invalidateDevDeps(invalidDevDeps, this.options, this.config); let {configs, bundleConfigs} = await this.loadConfigs(bundleGraph, bundle); let bundleInfo = (await this.getBundleInfoFromCache( bundleGraph, bundle, configs, bundleConfigs, )) ?? (await this.getBundleInfo(bundle, bundleGraph, configs, bundleConfigs)); let configRequests = getConfigRequests([ ...configs.values(), ...bundleConfigs.values(), ]); let devDepRequests = getWorkerDevDepRequests([ ...this.devDepRequests.values(), ]); return { bundleInfo, configRequests, devDepRequests, invalidations: [...this.invalidations.values()], }; } async loadConfigs( bundleGraph: InternalBundleGraph, bundle: InternalBundle, ): Promise<{ configs: Map; bundleConfigs: Map; }> { let configs = new Map(); let bundleConfigs = new Map(); await this.loadConfig(bundleGraph, bundle, configs, bundleConfigs); for (let inlineBundle of bundleGraph.getInlineBundles(bundle)) { await this.loadConfig(bundleGraph, inlineBundle, configs, bundleConfigs); } return {configs, bundleConfigs}; } async loadConfig( bundleGraph: InternalBundleGraph, bundle: InternalBundle, configs: Map, bundleConfigs: Map, ): Promise { let name = nullthrows(bundle.name); let plugin = await this.config.getPackager(name); await this.loadPluginConfig( bundleGraph, bundle, plugin, configs, bundleConfigs, ); let optimizers = await this.config.getOptimizers(name, bundle.pipeline); for (let optimizer of optimizers) { await this.loadPluginConfig( bundleGraph, bundle, optimizer, configs, bundleConfigs, ); } } async loadPluginConfig( bundleGraph: InternalBundleGraph, bundle: InternalBundle, plugin: LoadedPlugin, configs: Map, bundleConfigs: Map, ): Promise { if (!configs.has(plugin.name)) { // Only load config for a plugin once per build. let existing = pluginConfigs.get(plugin.name); if (existing != null) { // @ts-expect-error TS2345 configs.set(plugin.name, existing); } else { if (plugin.plugin.loadConfig != null) { let config = createConfig({ plugin: plugin.name, searchPath: toProjectPathUnsafe('index'), }); await loadPluginConfig(plugin, config, this.options); for (let devDep of config.devDeps) { let devDepRequest = await createDevDependency( devDep, this.previousDevDeps, this.options, ); let key = `${devDep.specifier}:${fromProjectPath( this.options.projectRoot, devDep.resolveFrom, )}`; this.devDepRequests.set(key, devDepRequest); } pluginConfigs.set(plugin.name, config); configs.set(plugin.name, config); } } } let loadBundleConfig = plugin.plugin.loadBundleConfig; if (!bundleConfigs.has(plugin.name) && loadBundleConfig != null) { let config = createConfig({ plugin: plugin.name, searchPath: joinProjectPath( bundle.target.distDir, bundle.name ?? bundle.id, ), }); config.result = await loadBundleConfig({ bundle: NamedBundle.get(bundle, bundleGraph, this.options), bundleGraph: new BundleGraph( bundleGraph, NamedBundle.get.bind(NamedBundle), this.options, ), config: new PublicConfig(config, this.options), options: new PluginOptions(this.options), logger: new PluginLogger({origin: plugin.name}), tracer: new PluginTracer({origin: plugin.name, category: 'loadConfig'}), }); bundleConfigs.set(plugin.name, config); } } async getBundleInfoFromCache( bundleGraph: InternalBundleGraph, bundle: InternalBundle, configs: Map, bundleConfigs: Map, ): Promise> { if (this.options.shouldDisableCache) { return; } let cacheKey = await this.getCacheKey( bundle, bundleGraph, configs, bundleConfigs, this.previousInvalidations, ); let infoKey = PackagerRunner.getInfoKey(cacheKey); return this.options.cache.get(infoKey); } async getBundleInfo( bundle: InternalBundle, bundleGraph: InternalBundleGraph, configs: Map, bundleConfigs: Map, ): Promise { let {type, contents, map, scopeHoistingStats} = await this.getBundleResult( bundle, bundleGraph, configs, bundleConfigs, ); // Recompute cache keys as they may have changed due to dev dependencies. let cacheKey = await this.getCacheKey( bundle, bundleGraph, configs, bundleConfigs, [...this.invalidations.values()], ); let cacheKeys = { content: PackagerRunner.getContentKey(cacheKey), map: PackagerRunner.getMapKey(cacheKey), info: PackagerRunner.getInfoKey(cacheKey), }; let cachedResult = await this.writeToCache(cacheKeys, type, contents, map); if (debugTools['scope-hoisting-stats']) { return {...cachedResult, scopeHoistingStats}; } return cachedResult; } async getBundleResult( bundle: InternalBundle, bundleGraph: InternalBundleGraph, configs: Map, bundleConfigs: Map, ): Promise<{ type: string; contents: Blob; map: string | null | undefined; scopeHoistingStats?: BundleResult['scopeHoistingStats']; }> { let packaged = await this.package( bundle, bundleGraph, configs, bundleConfigs, ); let type = packaged.type ?? bundle.type; let res = await this.optimize( bundle, bundleGraph, type, packaged.contents, packaged.map, configs, bundleConfigs, ); let map = res.map != null ? await this.generateSourceMap(bundle, res.map) : null; return { type: res.type ?? type, contents: res.contents, map, scopeHoistingStats: packaged.scopeHoistingStats, }; } getSourceMapReference( bundle: NamedBundle, map?: SourceMap | null, ): Async { if ( map && bundle.env.sourceMap && bundle.bundleBehavior !== 'inline' && bundle.bundleBehavior !== 'inlineIsolated' ) { if (bundle.env.sourceMap && bundle.env.sourceMap.inline) { return this.generateSourceMap(bundleToInternalBundle(bundle), map); } else { return path.basename(bundle.name) + '.map'; } } else { return null; } } async package( internalBundle: InternalBundle, bundleGraph: InternalBundleGraph, configs: Map, bundleConfigs: Map, ): Promise { let bundle = NamedBundle.get(internalBundle, bundleGraph, this.options); this.report({ type: 'buildProgress', phase: 'packaging', bundle, }); let packager = await this.config.getPackager(bundle.name); let {name, resolveFrom, plugin} = packager; let measurement; try { measurement = tracer.createMeasurement(name, 'packaging', bundle.name, { type: bundle.type, }); return await plugin.package({ config: configs.get(name)?.result, bundleConfig: bundleConfigs.get(name)?.result, bundle, bundleGraph: new BundleGraph( bundleGraph, NamedBundle.get.bind(NamedBundle), this.options, ), getSourceMapReference: (map) => { return this.getSourceMapReference(bundle, map); }, options: this.pluginOptions, logger: new PluginLogger({origin: name}), tracer: new PluginTracer({origin: name, category: 'package'}), getInlineBundleContents: async ( bundle: BundleType, bundleGraph: BundleGraphType, ) => { if ( bundle.bundleBehavior !== 'inline' && bundle.bundleBehavior !== 'inlineIsolated' ) { throw new Error( 'Bundle is not inline and unable to retrieve contents', ); } let {contents} = await this.getBundleResult( bundleToInternalBundle(bundle), bundleGraphToInternalBundleGraph(bundleGraph), configs, bundleConfigs, ); return {contents}; }, }); } catch (e: any) { throw new ThrowableDiagnostic({ diagnostic: errorToDiagnostic(e, { origin: name, filePath: path.join(bundle.target.distDir, bundle.name), }), }); } finally { measurement && measurement.end(); // Add dev dependency for the packager. This must be done AFTER running it due to // the potential for lazy require() that aren't executed until the request runs. let devDepRequest = await createDevDependency( { specifier: name, resolveFrom, }, this.previousDevDeps, this.options, ); this.devDepRequests.set( `${name}:${fromProjectPathRelative(resolveFrom)}`, devDepRequest, ); } } async optimize( internalBundle: InternalBundle, internalBundleGraph: InternalBundleGraph, type: string, contents: Blob, map: SourceMap | null | undefined, configs: Map, bundleConfigs: Map, ): Promise { let bundle = NamedBundle.get( internalBundle, internalBundleGraph, this.options, ); let bundleGraph = new BundleGraph( internalBundleGraph, NamedBundle.get.bind(NamedBundle), this.options, ); let optimizers = await this.config.getOptimizers( bundle.name, internalBundle.pipeline, ); if (!optimizers.length) { return {type: bundle.type, contents, map}; } this.report({ type: 'buildProgress', phase: 'optimizing', bundle, }); let optimized = { type, contents, map, }; for (let optimizer of optimizers) { let measurement; try { measurement = tracer.createMeasurement( optimizer.name, 'optimize', bundle.name, ); let next = await optimizer.plugin.optimize({ config: configs.get(optimizer.name)?.result, bundleConfig: bundleConfigs.get(optimizer.name)?.result, bundle, bundleGraph, contents: optimized.contents, map: optimized.map, getSourceMapReference: (map) => { return this.getSourceMapReference(bundle, map); }, options: this.pluginOptions, logger: new PluginLogger({origin: optimizer.name}), tracer: new PluginTracer({ origin: optimizer.name, category: 'optimize', }), }); optimized.type = next.type ?? optimized.type; optimized.contents = next.contents; optimized.map = next.map; } catch (e: any) { throw new ThrowableDiagnostic({ diagnostic: errorToDiagnostic(e, { origin: optimizer.name, filePath: path.join(bundle.target.distDir, bundle.name), }), }); } finally { measurement && measurement.end(); // Add dev dependency for the optimizer. This must be done AFTER running it due to // the potential for lazy require() that aren't executed until the request runs. let devDepRequest = await createDevDependency( { specifier: optimizer.name, resolveFrom: optimizer.resolveFrom, }, this.previousDevDeps, this.options, ); this.devDepRequests.set( `${optimizer.name}:${fromProjectPathRelative(optimizer.resolveFrom)}`, devDepRequest, ); } } return optimized; } async generateSourceMap( bundle: InternalBundle, map: SourceMap, ): Promise { let sourceRoot = computeSourceMapRoot(bundle, this.options); let inlineSources = sourceRoot === undefined; let filePath = joinProjectPath( bundle.target.distDir, nullthrows(bundle.name), ); let fullPath = fromProjectPath(this.options.projectRoot, filePath); let mapFilename = fullPath + '.map'; const bundleEnv = fromEnvironmentId(bundle.env); let isInlineMap = bundleEnv.sourceMap && bundleEnv.sourceMap.inline; let stringified = await map.stringify({ file: path.basename(mapFilename), fs: this.options.inputFS, rootDir: this.options.projectRoot, sourceRoot, inlineSources, format: isInlineMap ? 'inline' : 'string', }); invariant(typeof stringified === 'string'); return stringified; } async getCacheKey( bundle: InternalBundle, bundleGraph: InternalBundleGraph, configs: Map, bundleConfigs: Map, invalidations: Array, ): Promise { let configResults: Record = {}; for (let [pluginName, config] of configs) { if (config) { configResults[pluginName] = await getConfigHash( config, pluginName, this.options, ); } } let globalInfoResults: Record = {}; for (let [pluginName, config] of bundleConfigs) { if (config) { globalInfoResults[pluginName] = await getConfigHash( config, pluginName, this.options, ); } } let devDepHashes = await this.getDevDepHashes(bundle); for (let inlineBundle of bundleGraph.getInlineBundles(bundle)) { devDepHashes += await this.getDevDepHashes(inlineBundle); } let invalidationHash = await getInvalidationHash( invalidations, this.options, ); return hashString( ATLASPACK_VERSION + devDepHashes + invalidationHash + bundle.target.publicUrl + bundleGraph.getHash(bundle) + JSON.stringify(configResults) + JSON.stringify(globalInfoResults) + this.options.mode + (this.options.shouldBuildLazily ? 'lazy' : 'eager'), ); } async getDevDepHashes(bundle: InternalBundle): Promise { let name = nullthrows(bundle.name); let packager = await this.config.getPackager(name); let optimizers = await this.config.getOptimizers(name); let key = `${packager.name}:${fromProjectPathRelative( packager.resolveFrom, )}`; let devDepHashes = this.devDepRequests.get(key)?.hash ?? this.previousDevDeps.get(key) ?? ''; for (let {name, resolveFrom} of optimizers) { let key = `${name}:${fromProjectPathRelative(resolveFrom)}`; devDepHashes += this.devDepRequests.get(key)?.hash ?? this.previousDevDeps.get(key) ?? ''; } return devDepHashes; } async readFromCache(cacheKey: string): Promise< | { contents: Readable; map: Readable | null | undefined; } | null | undefined > { let contentKey = PackagerRunner.getContentKey(cacheKey); let mapKey = PackagerRunner.getMapKey(cacheKey); let isLargeBlob = await this.options.cache.hasLargeBlob(contentKey); let contentExists = isLargeBlob || (await this.options.cache.has(contentKey)); if (!contentExists) { return null; } let mapExists = await this.options.cache.has(mapKey); return { contents: isLargeBlob ? this.options.cache.getStream(contentKey) : blobToStream(await this.options.cache.getBlob(contentKey)), map: mapExists ? blobToStream(await this.options.cache.getBlob(mapKey)) : null, }; } async writeToCache( cacheKeys: CacheKeyMap, type: string, contents: Blob, map?: string | null, ): Promise { let size = 0; let hash; // @ts-expect-error TS2702 let hashReferences: RegExp.matchResult | Array = []; let isLargeBlob = false; // TODO: don't replace hash references in binary files?? if (contents instanceof Readable) { isLargeBlob = true; let boundaryStr = ''; let h = new Hash(); await this.options.cache.setStream( cacheKeys.content, blobToStream(contents).pipe( // @ts-expect-error TS2554 new TapStream((buf: Buffer) => { let str = boundaryStr + buf.toString(); hashReferences = hashReferences.concat( str.match(HASH_REF_REGEX) ?? [], ); size += buf.length; h.writeBuffer(buf); boundaryStr = str.slice(str.length - BOUNDARY_LENGTH); }), ), ); hash = h.finish(); } else if (typeof contents === 'string') { let buffer = Buffer.from(contents); size = buffer.byteLength; hash = hashBuffer(buffer); hashReferences = contents.match(HASH_REF_REGEX) ?? []; await this.options.cache.setBlob(cacheKeys.content, buffer); } else { size = contents.length; hash = hashBuffer(contents); hashReferences = contents.toString().match(HASH_REF_REGEX) ?? []; await this.options.cache.setBlob(cacheKeys.content, contents); } if (map != null) { await this.options.cache.setBlob(cacheKeys.map, map); } let info = { type, size, hash, hashReferences, cacheKeys, isLargeBlob, }; await this.options.cache.set(cacheKeys.info, info); return info; } static getContentKey(cacheKey: string): string { return hashString(`${cacheKey}:content`); } static getMapKey(cacheKey: string): string { return hashString(`${cacheKey}:map`); } static getInfoKey(cacheKey: string): string { return hashString(`${cacheKey}:info`); } }