// Dependency Installer // Handles the full install lifecycle: resolve, download, extract, transform, bin stubs, lock file. import { MemoryVolume } from "../memory-volume"; import { RegistryClient, RegistryConfig } from "./registry-client"; import { resolveDependencyTree, resolveFromManifest, ResolvedDependency, ResolutionConfig, } from "./version-resolver"; import { downloadAndExtract } from "./archive-extractor"; import { convertPackage, prepareTransformer } from "../module-transformer"; import type { PackageManifest } from "../types/manifest"; import * as path from "../polyfills/path"; import type { IDBSnapshotCache } from "../persistence/idb-cache"; import { quickDigest } from "../helpers/digest"; import { base64ToBytes } from "../helpers/byte-encoding"; import type { PrebundleStore } from "./prebundle/store"; // --------------------------------------------------------------------------- // Public types // --------------------------------------------------------------------------- export interface InstallFlags { registry?: string; persist?: boolean; persistDev?: boolean; withDevDeps?: boolean; withOptionalDeps?: boolean; onProgress?: (message: string) => void; // default: true transformModules?: boolean; } export interface InstallOutcome { resolved: Map; newPackages: string[]; } // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- // Normalize bin field — handles both shorthand string and object forms function normalizeBinField( packageName: string, bin?: string | Record, ): Record { if (!bin) return {}; if (typeof bin === "string") { const command = packageName.includes("/") ? packageName.split("/").pop()! : packageName; return { [command]: bin }; } return bin; } // Split "express@4.18.2" or "@types/node@20" into name + version function splitSpecifier(spec: string): { name: string; version?: string } { if (spec.startsWith("@")) { const slashIdx = spec.indexOf("/"); if (slashIdx === -1) throw new Error(`Malformed package specifier: ${spec}`); const tail = spec.slice(slashIdx + 1); const atIdx = tail.indexOf("@"); if (atIdx === -1) return { name: spec }; return { name: spec.slice(0, slashIdx + 1 + atIdx), version: tail.slice(atIdx + 1), }; } const atIdx = spec.indexOf("@"); if (atIdx === -1) return { name: spec }; return { name: spec.slice(0, atIdx), version: spec.slice(atIdx + 1), }; } // --------------------------------------------------------------------------- // Main class // --------------------------------------------------------------------------- let transformerReady = false; export class DependencyInstaller { private vol: MemoryVolume; private registryClient: RegistryClient; private workingDir: string; private _snapshotCache: IDBSnapshotCache | null; private _prebundleStore: PrebundleStore | null; constructor(vol: MemoryVolume, opts: { cwd?: string; snapshotCache?: IDBSnapshotCache | null; prebundleStore?: PrebundleStore | null } & RegistryConfig = {}) { this.vol = vol; this.registryClient = new RegistryClient(opts); this.workingDir = opts.cwd || "/"; this._snapshotCache = opts.snapshotCache ?? null; this._prebundleStore = opts.prebundleStore ?? null; } /** Set or replace the prebundle store after construction. */ setPrebundleStore(store: PrebundleStore | null): void { this._prebundleStore = store; } // ----------------------------------------------------------------------- // Public API // ----------------------------------------------------------------------- async install( packageName: string, version?: string, flags: InstallFlags = {}, ): Promise { const { onProgress } = flags; const spec = splitSpecifier(packageName); const targetName = spec.name; const targetRange = version || spec.version || "latest"; onProgress?.(`Resolving ${targetName}@${targetRange}...`); const resolutionOpts: ResolutionConfig = { registry: this.registryClient, devDependencies: flags.withDevDeps, optionalDependencies: flags.withOptionalDeps, onProgress, }; const tree = await resolveDependencyTree( targetName, targetRange, resolutionOpts, ); const newPkgs = await this.materializePackages(tree, flags); if (flags.persist || flags.persistDev) { const entry = tree.get(targetName); if (entry) { await this.patchManifest( targetName, `^${entry.version}`, !!flags.persistDev, ); } } onProgress?.(`Installed ${tree.size} package(s)`); return { resolved: tree, newPackages: newPkgs }; } async installFromManifest( manifestPath?: string, flags: InstallFlags = {}, ): Promise { const { onProgress } = flags; const jsonPath = manifestPath || path.join(this.workingDir, "package.json"); if (!this.vol.existsSync(jsonPath)) { throw new Error(`Manifest not found at ${jsonPath}`); } const raw = this.vol.readFileSync(jsonPath, "utf8"); const manifest: PackageManifest = JSON.parse(raw); // Check IDB snapshot cache — skip full install if we have a cached node_modules const cacheKey = this._snapshotCache ? quickDigest(raw) : null; console.log('[nodepod:installer] installFromManifest — snapshotCache:', !!this._snapshotCache, 'cacheKey:', cacheKey ? cacheKey.substring(0, 12) + '...' : 'null'); if (this._snapshotCache && cacheKey) { try { console.log('[nodepod:installer] Checking IDB cache for package.json hash...'); const tCache = performance.now(); const cached = await this._snapshotCache.get(cacheKey); if (cached) { onProgress?.("Restoring cached node_modules..."); console.log('[nodepod:installer] CACHE HIT — restoring', cached.entries.length, 'entries from IndexedDB'); const { entries } = cached; let dirs = 0, files = 0; // Restore only node_modules entries from the snapshot for (const entry of entries) { if (!entry.path.includes('/node_modules/')) continue; if (entry.kind === 'directory') { if (!this.vol.existsSync(entry.path)) { this.vol.mkdirSync(entry.path, { recursive: true }); } dirs++; } else if (entry.kind === 'file' && entry.data) { const parentDir = entry.path.substring(0, entry.path.lastIndexOf('/')) || '/'; if (parentDir !== '/' && !this.vol.existsSync(parentDir)) { this.vol.mkdirSync(parentDir, { recursive: true }); } this.vol.writeFileSync(entry.path, base64ToBytes(entry.data)); files++; } } const elapsed = (performance.now() - tCache).toFixed(0); console.log(`[nodepod:installer] CACHE RESTORE complete in ${elapsed}ms — ${dirs} dirs, ${files} files`); onProgress?.(`Restored ${entries.length} cached entries in ${elapsed}ms`); return { resolved: new Map(), newPackages: [] }; } else { console.log('[nodepod:installer] CACHE MISS — no cached snapshot, will install from registry'); } } catch (err) { console.warn('[nodepod:installer] Cache lookup failed, proceeding with install:', err); } } onProgress?.("Resolving dependency tree..."); console.log('[nodepod:installer] Resolving dependency tree from registry...'); const resolutionOpts: ResolutionConfig = { registry: this.registryClient, devDependencies: flags.withDevDeps, optionalDependencies: flags.withOptionalDeps, onProgress, }; const tree = await resolveFromManifest(manifest, resolutionOpts); console.log('[nodepod:installer] Resolved', tree.size, 'packages, materializing...'); const newPkgs = await this.materializePackages(tree, flags); console.log('[nodepod:installer] Materialized', newPkgs.length, 'new packages'); // Cache the installed node_modules snapshot for future reuse if (this._snapshotCache && cacheKey && newPkgs.length > 0) { try { console.log('[nodepod:installer] Caching node_modules snapshot to IndexedDB...'); const tSnapshot = performance.now(); const snapshot = this.vol.toSnapshot(); // Filter to only node_modules entries to keep cache lean const nmSnapshot = { entries: snapshot.entries.filter(e => e.path.includes('/node_modules/')), }; console.log('[nodepod:installer] Snapshot has', nmSnapshot.entries.length, 'node_modules entries (from', snapshot.entries.length, 'total)'); await this._snapshotCache.set(cacheKey, nmSnapshot); console.log(`[nodepod:installer] Snapshot cached in ${(performance.now() - tSnapshot).toFixed(0)}ms`); } catch (err) { console.warn('[nodepod:installer] Cache write failed:', err); } } else if (!this._snapshotCache) { console.log('[nodepod:installer] No snapshot cache available — skipping cache write'); } onProgress?.(`Installed ${tree.size} package(s)`); return { resolved: tree, newPackages: newPkgs }; } listInstalled(): Record { const nmDir = path.join(this.workingDir, "node_modules"); if (!this.vol.existsSync(nmDir)) return {}; const result: Record = {}; const topLevel = this.vol.readdirSync(nmDir) as string[]; for (const entry of topLevel) { if (entry.startsWith(".")) continue; if (entry.startsWith("@")) { const scopeDir = path.join(nmDir, entry); const scopedEntries = this.vol.readdirSync(scopeDir) as string[]; for (const child of scopedEntries) { const manifest = path.join(scopeDir, child, "package.json"); if (this.vol.existsSync(manifest)) { const data = JSON.parse(this.vol.readFileSync(manifest, "utf8")); result[`${entry}/${child}`] = data.version; } } } else { const manifest = path.join(nmDir, entry, "package.json"); if (this.vol.existsSync(manifest)) { const data = JSON.parse(this.vol.readFileSync(manifest, "utf8")); result[entry] = data.version; } } } return result; } // ----------------------------------------------------------------------- // Private helpers // ----------------------------------------------------------------------- // Download, extract, transform, and wire up packages not already in node_modules private async materializePackages( tree: Map, flags: InstallFlags, ): Promise { const { onProgress } = flags; const additions: string[] = []; const nmRoot = path.join(this.workingDir, "node_modules"); this.vol.mkdirSync(nmRoot, { recursive: true }); const pending: Array<{ depName: string; dep: ResolvedDependency; targetDir: string; }> = []; // Phase 1: Satisfy as many packages as possible from the prebundle store const prebundled: string[] = []; for (const [depName, dep] of tree) { const targetDir = path.join(nmRoot, depName); const existingManifest = path.join(targetDir, "package.json"); if (this.vol.existsSync(existingManifest)) { try { const current = JSON.parse( this.vol.readFileSync(existingManifest, "utf8"), ); if (current.version === dep.version) { onProgress?.(`Skipping ${depName}@${dep.version} (up to date)`); continue; } } catch { // corrupt manifest, reinstall } } // Check prebundle store before queueing for download if (this._prebundleStore?.has(depName, dep.version)) { onProgress?.(` Loading ${depName}@${dep.version} (prebundled)`); const count = this._prebundleStore.materialize(depName, this.vol, nmRoot); if (count > 0) { this.createBinStubs(nmRoot, depName, targetDir); prebundled.push(depName); additions.push(depName); continue; } } pending.push({ depName, dep, targetDir }); } if (prebundled.length > 0) { onProgress?.(`Loaded ${prebundled.length} prebundled package(s)`); } // Only need main-thread transformer as fallback when workers aren't available const shouldTransform = flags.transformModules !== false; if (shouldTransform && !transformerReady) { if (typeof Worker === "undefined") { onProgress?.("Preparing module transformer..."); await prepareTransformer(); } transformerReady = true; } // Safe to batch aggressively since extract + transform are offloaded to workers const WORKER_COUNT = 12; onProgress?.(`Downloading ${pending.length} package(s)...`); for (let offset = 0; offset < pending.length; offset += WORKER_COUNT) { const batch = pending.slice(offset, offset + WORKER_COUNT); await Promise.all( batch.map(async ({ depName, dep, targetDir }) => { onProgress?.(` Fetching ${depName}@${dep.version}...`); await downloadAndExtract(dep.tarballUrl, this.vol, targetDir, { stripComponents: 1, expectedShasum: dep.shasum, }); if (shouldTransform) { try { const transformed = await convertPackage( this.vol, targetDir, onProgress, ); if (transformed > 0) { onProgress?.( ` Transformed ${transformed} file(s) in ${depName}`, ); } } catch (err) { onProgress?.( ` Warning: transformation failed for ${depName}: ${err}`, ); } } this.createBinStubs(nmRoot, depName, targetDir); additions.push(depName); }), ); } this.writeLockFile(tree); return additions; } private createBinStubs( nmRoot: string, depName: string, pkgDir: string, ): void { try { const manifestPath = path.join(pkgDir, "package.json"); if (!this.vol.existsSync(manifestPath)) return; const data = JSON.parse(this.vol.readFileSync(manifestPath, "utf8")); const bins = normalizeBinField(depName, data.bin); const binDir = path.join(nmRoot, ".bin"); for (const [cmd, relPath] of Object.entries(bins)) { this.vol.mkdirSync(binDir, { recursive: true }); const target = path.join(pkgDir, relPath); this.vol.writeFileSync( path.join(binDir, cmd), `node "${target}" "$@"\n`, ); } } catch { // best-effort } } private writeLockFile(tree: Map): void { const entries: Record = {}; for (const [depName, dep] of tree) { entries[depName] = { version: dep.version, resolved: dep.tarballUrl, }; } const lockPath = path.join( this.workingDir, "node_modules", ".package-lock.json", ); this.vol.writeFileSync(lockPath, JSON.stringify(entries, null, 2)); } private async patchManifest( depName: string, versionSpec: string, asDev: boolean, ): Promise { const jsonPath = path.join(this.workingDir, "package.json"); let manifest: Record = {}; if (this.vol.existsSync(jsonPath)) { manifest = JSON.parse(this.vol.readFileSync(jsonPath, "utf8")); } const section = asDev ? "devDependencies" : "dependencies"; if (!manifest[section]) { manifest[section] = {}; } (manifest[section] as Record)[depName] = versionSpec; this.vol.writeFileSync(jsonPath, JSON.stringify(manifest, null, 2)); } } // --------------------------------------------------------------------------- // Convenience function // --------------------------------------------------------------------------- // One-shot install: `install("express@4.18.2", vol)` export async function install( specifier: string, vol: MemoryVolume, flags?: InstallFlags, ): Promise { const installer = new DependencyInstaller(vol); return installer.install(specifier, undefined, flags); } export { RegistryClient } from "./registry-client"; export type { RegistryConfig, VersionDetail, PackageMetadata, } from "./registry-client"; export type { ResolvedDependency, ResolutionConfig } from "./version-resolver"; export type { ExtractionOptions } from "./archive-extractor"; export { splitSpecifier };