import type {NodeId} from '@atlaspack/graph'; import type {Async} from '@atlaspack/types'; import type {SharedReference} from '@atlaspack/workers'; import type { Asset, AssetGroup, AssetRequestInput, Dependency, Entry, AtlaspackOptions, Target, } from '../types'; import type {StaticRunOpts, RunAPI} from '../RequestTracker'; import type {EntryRequestResult} from './EntryRequest'; import type {PathRequestInput} from './PathRequest'; import type {Diagnostic} from '@atlaspack/diagnostic'; import logger from '@atlaspack/logger'; import invariant from 'assert'; import nullthrows from 'nullthrows'; import {PromiseQueue, setEqual} from '@atlaspack/utils'; import {hashString} from '@atlaspack/rust'; import ThrowableDiagnostic from '@atlaspack/diagnostic'; import {Priority} from '../types'; import AssetGraph from '../AssetGraph'; import {ATLASPACK_VERSION} from '../constants'; import createEntryRequest from './EntryRequest'; import createTargetRequest from './TargetRequest'; import createAssetRequest from './AssetRequest'; import createPathRequest from './PathRequest'; import {ProjectPath, fromProjectPathRelative} from '../projectPath'; import dumpGraphToGraphViz from '../dumpGraphToGraphViz'; import {propagateSymbols} from '../SymbolPropagation'; import {requestTypes} from '../RequestTracker'; export type AssetGraphRequestInput = { entries?: Array; assetGroups?: Array; optionsRef: SharedReference; name: string; shouldBuildLazily?: boolean; lazyIncludes?: RegExp[]; lazyExcludes?: RegExp[]; requestedAssetIds?: Set; skipSymbolProp?: boolean; }; export type AssetGraphRequestResult = { assetGraph: AssetGraph; /** Assets added/modified since the last successful build. */ changedAssets: Map; /** Assets added/modified since the last symbol propagation invocation. */ changedAssetsPropagation: Set; assetGroupsWithRemovedParents: Set | null | undefined; previousSymbolPropagationErrors: | Map> | null | undefined; assetRequests: Array; }; type RunInput = { input: AssetGraphRequestInput; } & StaticRunOpts; type AssetGraphRequest = { id: string; readonly type: typeof requestTypes.asset_graph_request; run: (arg1: RunInput) => Async; input: AssetGraphRequestInput; }; export default function createAssetGraphRequest( requestInput: AssetGraphRequestInput, ): AssetGraphRequest { return { type: requestTypes.asset_graph_request, id: requestInput.name, run: async (input) => { let prevResult = await input.api.getPreviousResult(); let builder = new AssetGraphBuilder(input, prevResult); let assetGraphRequest = await builder.build(); // early break for incremental bundling if production or flag is off; if ( !input.options.shouldBundleIncrementally || input.options.mode === 'production' ) { assetGraphRequest.assetGraph.safeToIncrementallyBundle = false; } if ( !input.options.shouldBundleIncrementally || input.options.mode === 'production' ) { assetGraphRequest.assetGraph.safeToIncrementallyBundle = false; } return assetGraphRequest; }, input: requestInput, }; } const typesWithRequests = new Set([ 'entry_specifier', 'entry_file', 'dependency', 'asset_group', ]); export class AssetGraphBuilder { assetGraph: AssetGraph; assetRequests: Array = []; queue: PromiseQueue; changedAssets: Map; changedAssetsPropagation: Set; prevChangedAssetsPropagation: Set | null | undefined; optionsRef: SharedReference; options: AtlaspackOptions; api: RunAPI; name: string; cacheKey: string; shouldBuildLazily: boolean; lazyIncludes: RegExp[]; lazyExcludes: RegExp[]; requestedAssetIds: Set; isSingleChangeRebuild: boolean; assetGroupsWithRemovedParents: Set; previousSymbolPropagationErrors: Map>; skipSymbolProp: boolean; constructor( {input, api, options}: RunInput, prevResult?: AssetGraphRequestResult | null, ) { let { entries, assetGroups, optionsRef, name, requestedAssetIds, shouldBuildLazily, lazyIncludes, lazyExcludes, } = input; let assetGraph = prevResult?.assetGraph ?? new AssetGraph(); assetGraph.safeToIncrementallyBundle = true; assetGraph.setRootConnections({ entries, assetGroups, }); assetGraph.undeferredDependencies.clear(); this.assetGroupsWithRemovedParents = prevResult?.assetGroupsWithRemovedParents ?? new Set(); this.previousSymbolPropagationErrors = prevResult?.previousSymbolPropagationErrors ?? new Map(); this.changedAssets = prevResult?.changedAssets ?? new Map(); this.changedAssetsPropagation = new Set(); this.prevChangedAssetsPropagation = prevResult?.changedAssetsPropagation; this.assetGraph = assetGraph; this.optionsRef = optionsRef; this.options = options; this.api = api; this.name = name; this.requestedAssetIds = requestedAssetIds ?? new Set(); this.shouldBuildLazily = shouldBuildLazily ?? false; this.lazyIncludes = lazyIncludes ?? []; this.lazyExcludes = lazyExcludes ?? []; this.skipSymbolProp = input.skipSymbolProp ?? false; this.cacheKey = hashString( `${ATLASPACK_VERSION}${name}${JSON.stringify(entries) ?? ''}${ options.mode }${options.shouldBuildLazily ? 'lazy' : 'eager'}`, ) + '-AssetGraph'; this.isSingleChangeRebuild = api .getInvalidSubRequests() // @ts-expect-error TS2367 .filter((req) => req.requestType === 'asset_request').length === 1; this.queue = new PromiseQueue(); assetGraph.onNodeRemoved = (nodeId: NodeId) => { this.assetGroupsWithRemovedParents.delete(nodeId); // This needs to mark all connected nodes that doesn't become orphaned // due to replaceNodesConnectedTo to make sure that the symbols of // nodes from which at least one parent was removed are updated. let node = nullthrows(assetGraph.getNode(nodeId)); if (assetGraph.isOrphanedNode(nodeId) && node.type === 'dependency') { let children = assetGraph.getNodeIdsConnectedFrom(nodeId); for (let child of children) { let childNode = nullthrows(assetGraph.getNode(child)); invariant( childNode.type === 'asset_group' || childNode.type === 'asset', ); childNode.usedSymbolsDownDirty = true; this.assetGroupsWithRemovedParents.add(child); } } }; } async build(): Promise { let errors: Array = []; let rootNodeId = nullthrows( this.assetGraph.rootNodeId, 'A root node is required to traverse', ); let visitedAssetGroups = new Set(); let visited = new Set([rootNodeId]); const visit = (nodeId: NodeId) => { if (errors.length > 0) { return; } if (this.shouldSkipRequest(nodeId)) { visitChildren(nodeId); } else { // ? do we need to visit children inside of the promise that is queued? this.queueCorrespondingRequest(nodeId, errors).then(() => visitChildren(nodeId), ); } }; const visitChildren = (nodeId: NodeId) => { for (let childNodeId of this.assetGraph.getNodeIdsConnectedFrom(nodeId)) { let child = nullthrows(this.assetGraph.getNode(childNodeId)); if ( // @ts-expect-error TS2339 (!visited.has(childNodeId) || child.hasDeferred) && this.shouldVisitChild(nodeId, childNodeId) ) { if (child.type === 'asset_group') { visitedAssetGroups.add(childNodeId); } visited.add(childNodeId); visit(childNodeId); } } }; visit(rootNodeId); await this.queue.run(); logger.verbose({ origin: '@atlaspack/core', message: 'Asset graph walked', meta: { visitedAssetGroupsCount: visitedAssetGroups.size, }, }); if (this.prevChangedAssetsPropagation) { // Add any previously seen Assets that have not been propagated yet to // 'this.changedAssetsPropagation', but only if they still remain in the graph // as they could have been removed since the last build for (let assetId of this.prevChangedAssetsPropagation) { if (this.assetGraph.hasContentKey(assetId)) { this.changedAssetsPropagation.add(assetId); } } } if (errors.length) { this.api.storeResult( { assetGraph: this.assetGraph, changedAssets: this.changedAssets, changedAssetsPropagation: this.changedAssetsPropagation, assetGroupsWithRemovedParents: this.assetGroupsWithRemovedParents, previousSymbolPropagationErrors: undefined, assetRequests: [], }, this.cacheKey, ); // TODO: eventually support multiple errors since requests could reject in parallel throw errors[0]; } if (this.assetGraph.nodes.length > 1) { await dumpGraphToGraphViz( this.assetGraph, 'AssetGraph_' + this.name + '_before_prop', ); // Skip symbol propagation for runtime assets - they have pre-computed symbol data if (this.skipSymbolProp) { logger.verbose({ origin: '@atlaspack/core', message: 'Skipping symbol propagation for runtime asset graph', }); } else { try { let errors = propagateSymbols({ options: this.options, assetGraph: this.assetGraph, changedAssetsPropagation: this.changedAssetsPropagation, assetGroupsWithRemovedParents: this.assetGroupsWithRemovedParents, previousErrors: this.previousSymbolPropagationErrors, }); this.changedAssetsPropagation.clear(); if (errors.size > 0) { this.api.storeResult( { assetGraph: this.assetGraph, changedAssets: this.changedAssets, changedAssetsPropagation: this.changedAssetsPropagation, assetGroupsWithRemovedParents: this.assetGroupsWithRemovedParents, previousSymbolPropagationErrors: errors, assetRequests: [], }, this.cacheKey, ); // Just throw the first error. Since errors can bubble (e.g. reexporting a reexported symbol also fails), // determining which failing export is the root cause is nontrivial (because of circular dependencies). throw new ThrowableDiagnostic({ diagnostic: [...errors.values()][0], }); } } catch (e: any) { await dumpGraphToGraphViz( this.assetGraph, 'AssetGraph_' + this.name + '_failed', ); throw e; } } } await dumpGraphToGraphViz(this.assetGraph, 'AssetGraph_' + this.name); this.api.storeResult( { assetGraph: this.assetGraph, changedAssets: new Map(), changedAssetsPropagation: this.changedAssetsPropagation, assetGroupsWithRemovedParents: undefined, previousSymbolPropagationErrors: undefined, assetRequests: [], }, this.cacheKey, ); return { assetGraph: this.assetGraph, changedAssets: this.changedAssets, changedAssetsPropagation: this.changedAssetsPropagation, assetGroupsWithRemovedParents: undefined, previousSymbolPropagationErrors: undefined, assetRequests: this.assetRequests, }; } shouldVisitChild(nodeId: NodeId, childNodeId: NodeId): boolean { if (this.shouldBuildLazily) { let node = nullthrows(this.assetGraph.getNode(nodeId)); let childNode = nullthrows(this.assetGraph.getNode(childNodeId)); if (node.type === 'asset' && childNode.type === 'dependency') { // This logic will set `node.requested` to `true` if the node is in the list of requested asset ids // (i.e. this is an entry of a (probably) placeholder bundle that wasn't previously requested) // // Otherwise, if this node either is explicitly not requested, or has had it's requested attribute deleted, // it will determine whether this node is an "async child" - that is, is it a (probably) // dynamic import(). If so, it will explicitly have it's `node.requested` set to `false` // // If it's not requested, but it's not an async child then it's `node.requested` is deleted (undefined) // by default with lazy compilation all nodes are lazy let isNodeLazy = true; // For conditional lazy building - if this node matches the `lazyInclude` globs that means we want // only those nodes to be treated as lazy - that means if this node does _NOT_ match that glob, then we // also consider it not lazy (so it gets marked as requested). const relativePath = fromProjectPathRelative(node.value.filePath); if (this.lazyIncludes.length > 0) { isNodeLazy = this.lazyIncludes.some((lazyIncludeRegex) => relativePath.match(lazyIncludeRegex), ); } // Excludes override includes, so a node is _not_ lazy if it is included in the exclude list. if (this.lazyExcludes.length > 0 && isNodeLazy) { isNodeLazy = !this.lazyExcludes.some((lazyExcludeRegex) => relativePath.match(lazyExcludeRegex), ); } if (this.requestedAssetIds.has(node.value.id) || !isNodeLazy) { node.requested = true; } else if (!node.requested) { let isAsyncChild = this.assetGraph .getIncomingDependencies(node.value) .every((dep) => dep.isEntry || dep.priority !== Priority.sync); if ( isAsyncChild && childNode.value.priority !== Priority.conditional ) { // Skip if we're on a conditional import node.requested = !isNodeLazy; } else { delete node.requested; } } let previouslyDeferred = childNode.deferred; childNode.deferred = node.requested === false; // The child dependency node we're now evaluating should not be deferred if it's parent // is explicitly not requested (requested = false, but not requested = undefined) // // if we weren't previously deferred but we are now, then this dependency node's parents should also // be marked as deferred // // if we were previously deferred but we not longer are, then then all parents should no longer be // deferred either if (!previouslyDeferred && childNode.deferred) { this.assetGraph.markParentsWithHasDeferred(childNodeId); } else if (previouslyDeferred && !childNode.deferred) { // Mark Asset and Dependency as dirty for symbol propagation as it was // previously deferred and it's used symbols may have changed this.changedAssetsPropagation.add(node.id); node.usedSymbolsDownDirty = true; this.changedAssetsPropagation.add(childNode.id); childNode.usedSymbolsDownDirty = true; this.assetGraph.unmarkParentsWithHasDeferred(childNodeId); } // We `shouldVisitChild` if the childNode is not deferred return !childNode.deferred; } } return this.assetGraph.shouldVisitChild(nodeId, childNodeId); } shouldSkipRequest(nodeId: NodeId): boolean { let node = nullthrows(this.assetGraph.getNode(nodeId)); return ( // @ts-expect-error TS2339 node.complete === true || !typesWithRequests.has(node.type) || // @ts-expect-error TS2339 (node.correspondingRequest != null && // @ts-expect-error TS2339 this.api.canSkipSubrequest(node.correspondingRequest)) ); } queueCorrespondingRequest( nodeId: NodeId, errors: Array, ): Promise { let promise; let node = nullthrows(this.assetGraph.getNode(nodeId)); switch (node.type) { case 'entry_specifier': promise = this.runEntryRequest(node.value); break; case 'entry_file': promise = this.runTargetRequest(node.value); break; case 'dependency': promise = this.runPathRequest(node.value); break; case 'asset_group': promise = this.runAssetRequest(node.value); break; default: throw new Error( `Can not queue corresponding request of node with type ${node.type}`, ); } return new Promise((resolve: (result: Promise | null) => void) => { this.queue.add(() => promise.then( () => { resolve(null); }, (error) => errors.push(error), ), ); }); } async runEntryRequest(input: ProjectPath) { let prevEntries = this.assetGraph.safeToIncrementallyBundle ? this.assetGraph .getEntryAssets() .map((asset) => asset.id) .sort() : []; let request = createEntryRequest(input); let result = await this.api.runRequest( request, { force: true, }, ); this.assetGraph.resolveEntry(request.input, result.entries, request.id); if (this.assetGraph.safeToIncrementallyBundle) { let currentEntries = this.assetGraph .getEntryAssets() .map((asset) => asset.id) .sort(); let didEntriesChange = prevEntries.length !== currentEntries.length || prevEntries.every( (entryId, index) => entryId === currentEntries[index], ); if (didEntriesChange) { this.assetGraph.safeToIncrementallyBundle = false; } } } async runTargetRequest(input: Entry) { let request = createTargetRequest(input); let targets = await this.api.runRequest>(request, { force: true, }); this.assetGraph.resolveTargets(request.input, targets, request.id); } async runPathRequest(input: Dependency) { let request = createPathRequest({dependency: input, name: this.name}); let result = await this.api.runRequest< PathRequestInput, AssetGroup | null | undefined >(request, {force: true}); this.assetGraph.resolveDependency(input, result, request.id); } async runAssetRequest(input: AssetGroup) { this.assetRequests.push(input); // @ts-expect-error TS2345 let request = createAssetRequest({ ...input, name: this.name, optionsRef: this.optionsRef, isSingleChangeRebuild: this.isSingleChangeRebuild, }); let assets = await this.api.runRequest>( request, {force: true}, ); if (assets != null) { for (let asset of assets) { // Pass the runtimeAssetRequiringExecutionOnLoad flag from the asset // group down to the asset itself, for reading in the packager. if (input.runtimeAssetRequiringExecutionOnLoad) { asset.meta = { ...(asset.meta ?? {}), runtimeAssetRequiringExecutionOnLoad: input.runtimeAssetRequiringExecutionOnLoad, }; } if (this.assetGraph.safeToIncrementallyBundle) { let otherAsset = this.assetGraph.getNodeByContentKey(asset.id); if (otherAsset != null) { invariant(otherAsset.type === 'asset'); if (!this._areDependenciesEqualForAssets(asset, otherAsset.value)) { this.assetGraph.safeToIncrementallyBundle = false; } } else { // adding a new entry or dependency this.assetGraph.safeToIncrementallyBundle = false; } } this.changedAssets.set(asset.id, asset); this.changedAssetsPropagation.add(asset.id); } this.assetGraph.resolveAssetGroup(input, assets, request.id); } else { this.assetGraph.safeToIncrementallyBundle = false; } this.isSingleChangeRebuild = false; } /** * Used for incremental bundling of modified assets */ _areDependenciesEqualForAssets(asset: Asset, otherAsset: Asset): boolean { let assetDependencies = Array.from(asset?.dependencies.keys()).sort(); let otherAssetDependencies = Array.from( otherAsset?.dependencies.keys(), ).sort(); if (assetDependencies.length !== otherAssetDependencies.length) { return false; } return assetDependencies.every((key, index) => { if (key !== otherAssetDependencies[index]) { return false; } return setEqual( new Set(asset?.dependencies.get(key)?.symbols?.keys()), new Set(otherAsset?.dependencies.get(key)?.symbols?.keys()), ); }); } }