import { createHash } from 'crypto' import * as fs from 'fs/promises' import * as path from 'path' import { unzipSync, zipSync } from 'fflate' import ignore from 'ignore' import semver from 'semver' import { ALWAYS_IGNORED_DIRS, GITIGNORE_FILENAME, isGitignored, findInstallRoot, walkWithGitignore, type GitignoreMatcher, } from './project-walk.js' import type { MarketClient } from './client.js' import { lookupAssets } from './refs.js' import type { AssetInstallMetadata } from './contract.js' import { packageJsonAssetDependencies, packageJsonAssetDependenciesFromFiles, packageJsonNpmDependenciesFromFiles, parsePackageJson, } from './package-json.js' import { MAX_UPLOAD_ZIP_SIZE_BYTES, assetDependencyAlias, assetDependencyRange, assetDependencyValue, semverSchema, type AssetDependencies, type AssetDependencyAlias, type AssetType, } from './schemas.js' export interface PackDependencySpecs { npm?: string[] asset?: string[] skill?: string[] } export interface ParsedPackDependencies { npmDependencies: Record assetDependencies: Record skillDependencies: Record } export interface PackPolicy { readPackageJsonDependencies: boolean omitUnchangedInstalledFiles: boolean } export type FetchFileManifest = (name: string, range: string) => Promise> export interface PackAssetOptions { cwd?: string dependencies: ParsedPackDependencies policy?: PackPolicy /** * Fetch `path → etag` (md5) for a dependency's files given the declared range (see * `fileManifestForRange`). Pack matches local files against these hashes by content — * independent of path, so renamed installed files are omitted too and recorded as aliases. * Omit this (or a failing fetch) and the dependency's files are kept. */ fetchFileManifest?: FetchFileManifest } export interface PackedAsset { sourcePath: string zip: Uint8Array npmDependencies: Record assetDependencies: AssetDependencies skillDependencies: Record omittedUnchangedInstalledFiles: boolean /** Number of zip entries dropped because a `.gitignore` inside the zip excluded them. */ gitignoredFiles: number /** Dependencies whose file index could not be fetched: files kept, aliases untouched. */ skippedDependencies: string[] /** Dependency files whose canonical bytes were found nowhere (locally modified or deleted). */ missingDependencyFiles: Array<{ name: string; path: string }> } /** md5 hex — the same digest R2 serves as every file's etag, so local bytes match server hashes. */ export function md5(bytes: Uint8Array): string { return createHash('md5').update(bytes).digest('hex') } /** * Cap on how many in-range versions contribute hashes to one dependency's manifest. High enough * that any real project's installed version is covered; bounds the API calls for assets with long * histories (an out-of-cap version's files simply stay unmatched and ship — the fail-open * direction). */ const MANIFEST_VERSION_CAP = 20 /** * A `FetchFileManifest` backed by the Market API: `path → etag` (md5) per dependency file. Without * a local lockfile the installed version is unknown, so a non-exact range merges the file indexes * of every published version satisfying it (newest wins per path) — a hash match identifies the * file regardless of which version it came from. Content addressing keeps mistakes safe: a stale * hash simply fails to match and the file is kept. */ export function fileManifestForRange(asset: MarketClient['asset']): FetchFileManifest { return async (name, range) => { const exact = exactRangeVersion(range) const inRange = exact ? [exact] : ((await lookupAssets(asset, [{ name }]))[0]?.versions ?? []) .filter((version) => semver.satisfies(version, range)) .sort(semver.compare) .slice(-MANIFEST_VERSION_CAP) // Oldest first, so a newer version's hash wins when a path recurs across versions. const entries = await lookupAssets( asset, inRange.map((version) => ({ name, version })), ) const merged: Record = {} for (const entry of entries) { Object.assign( merged, Object.fromEntries((entry?.files ?? []).map((file) => [file.path, file.etag])), ) } return merged } } function exactRangeVersion(range: string): string | null { const candidate = range.trim().replace(/^=/u, '') return semverSchema.safeParse(candidate).success ? candidate : null } export async function packAsset(zipFilter: string, opts: PackAssetOptions): Promise { const cwd = opts.cwd ?? process.cwd() // A directory packs without ever zipping the full app first: gitignored trees are never read, // and unchanged installed dependency files are dropped by hash before compression — so a // template carrying hundreds of MB of reinstallable assets packs in seconds to a few MB. const direct = await maybeStat(path.resolve(cwd, zipFilter)) if (direct?.isDirectory()) return packDirectory(path.resolve(cwd, zipFilter), opts) const zipFile = await resolveOneZipFile(cwd, zipFilter) const zipStat = await fs.stat(zipFile) if (zipStat.size >= MAX_UPLOAD_ZIP_SIZE_BYTES) { throw new Error('Packed zip must be smaller than 1 GB') } const sourceZip = new Uint8Array(await fs.readFile(zipFile)) const sourceFiles = unzipSync(sourceZip) const policy = opts.policy ?? inferPackPolicy(sourceFiles) const packageJsonAssetDependencies = policy.readPackageJsonDependencies ? packageJsonAssetDependenciesFromFiles(sourceFiles) : {} const assetDependencies = mergeAssetDependencies( packageJsonAssetDependencies, opts.dependencies.assetDependencies, ) const packageJsonNpmDependencies = policy.readPackageJsonDependencies ? packageJsonNpmDependenciesFromFiles(sourceFiles) : {} const npmDependencies = { ...packageJsonNpmDependencies, ...opts.dependencies.npmDependencies } // Drop anything the zip's own .gitignore excludes (node_modules, build output, secrets…), then — // for templates — omit each installed dependency file at its resolved location (matched by // content hash, so renamed copies are dropped too and recorded as aliases; duplicates ship). // Only removals happen, so a smaller count means something was dropped; re-zip only then. const { kept: withoutIgnored, gitignored } = applyGitignore(sourceFiles) let files = withoutIgnored let finalAssetDependencies = assetDependencies let skippedDependencies: string[] = [] let missingDependencyFiles: Array<{ name: string; path: string }> = [] if (policy.omitUnchangedInstalledFiles) { const omitted = await withoutInstalledDependencyFiles( withoutIgnored, assetDependencies, opts.fetchFileManifest, ) files = omitted.kept finalAssetDependencies = omitted.assetDependencies skippedDependencies = omitted.skippedDependencies missingDependencyFiles = omitted.missingDependencyFiles } const originalCount = Object.keys(sourceFiles).length const afterIgnoreCount = Object.keys(withoutIgnored).length const finalCount = Object.keys(files).length return { sourcePath: zipFile, zip: finalCount < originalCount ? zipSync(files) : sourceZip, npmDependencies, assetDependencies: finalAssetDependencies, skillDependencies: opts.dependencies.skillDependencies, omittedUnchangedInstalledFiles: finalCount < afterIgnoreCount, gitignoredFiles: gitignored, skippedDependencies, missingDependencyFiles, } } // Pack straight from a project directory. Walk order matters for speed: `.gitignore` rules prune // whole subtrees before anything is read, and only the survivors are ever held in memory / // compressed. Installed dependency files are matched by content hash — independent of their path — // so renamed copies are omitted too (exactly one per dependency file; duplicates ship). The // resolved location is recorded as an alias on the dependency entry and written back to // package.json, which is how a later install restores the renamed layout. async function packDirectory(dir: string, opts: PackAssetOptions): Promise { const hasRootPackageJson = Boolean(await maybeStat(path.join(dir, 'package.json'))) const policy = opts.policy ?? { readPackageJsonDependencies: hasRootPackageJson, omitUnchangedInstalledFiles: hasRootPackageJson, } const walk = await walkWithGitignore(dir) const packageJsonFiles: Record = {} if (policy.readPackageJsonDependencies && hasRootPackageJson) { packageJsonFiles['package.json'] = new Uint8Array( await fs.readFile(path.join(dir, 'package.json')), ) } const assetDependencies = mergeAssetDependencies( policy.readPackageJsonDependencies ? packageJsonAssetDependenciesFromFiles(packageJsonFiles) : {}, opts.dependencies.assetDependencies, ) const npmDependencies = { ...(policy.readPackageJsonDependencies ? packageJsonNpmDependenciesFromFiles(packageJsonFiles) : {}), ...opts.dependencies.npmDependencies, } const index = policy.omitUnchangedInstalledFiles ? await dependencyFileIndex(assetDependencies, opts.fetchFileManifest) : emptyDependencyFileIndex() const dependencyHashes = new Set(index.files.map((file) => file.hash)) // Files whose bytes match a dependency file are deferred: whether one is omitted or ships (a // duplicate copy) is only known once the whole tree is hashed, and deferring keeps dependency // bytes out of memory during the walk. const files: Record = {} const hashesByPath = new Map() const deferred: string[] = [] for (const relative of walk.kept) { const content = new Uint8Array(await fs.readFile(path.join(dir, relative))) if (dependencyHashes.size === 0) { files[relative] = content continue } const hash = md5(content) hashesByPath.set(relative, hash) if (dependencyHashes.has(hash)) { deferred.push(relative) continue } files[relative] = content } const layout = resolveDependencyLayout(assetDependencies, index, hashesByPath) for (const relative of deferred) { if (layout.omit.has(relative)) continue files[relative] = new Uint8Array(await fs.readFile(path.join(dir, relative))) } if (policy.readPackageJsonDependencies && hasRootPackageJson) { const rewritten = await writeBackAssetDependencies(dir, layout.assetDependencies) if (rewritten && files['package.json']) files['package.json'] = rewritten } const zip = zipSync(files) if (zip.byteLength >= MAX_UPLOAD_ZIP_SIZE_BYTES) { throw new Error('Packed zip must be smaller than 1 GB') } return { sourcePath: dir, zip, npmDependencies, assetDependencies: layout.assetDependencies, skillDependencies: opts.dependencies.skillDependencies, omittedUnchangedInstalledFiles: layout.omit.size > 0, gitignoredFiles: walk.ignored, skippedDependencies: policy.omitUnchangedInstalledFiles ? Object.keys(assetDependencies).filter((name) => !index.fetched.has(name)) : [], missingDependencyFiles: layout.missing.map((file) => ({ name: file.name, path: file.canonicalPath, })), } } function hasAlwaysIgnoredSegment(posixPath: string): boolean { return posixPath.split('/').some((segment) => ALWAYS_IGNORED_DIRS.has(segment)) } /** * Filter zip entries: always drop the structural dirs above, then drop whatever a `.gitignore` * inside the zip excludes. `.gitignore` files apply per-directory (git semantics: a nested one only * affects its own subtree). Returns the survivors plus the count of `.gitignore`-matched drops. */ function applyGitignore(files: Record): { kept: Record gitignored: number } { const matchers = gitignoreMatchers(files) const kept: Record = {} let gitignored = 0 for (const [name, content] of Object.entries(files)) { if (hasAlwaysIgnoredSegment(name)) continue if (matchers.length > 0 && isGitignored(name, matchers)) { gitignored += 1 continue } kept[name] = content } return { kept, gitignored } } function gitignoreMatchers(files: Record): GitignoreMatcher[] { const matchers: GitignoreMatcher[] = [] for (const [name, content] of Object.entries(files)) { if (path.posix.basename(name) !== GITIGNORE_FILENAME) continue const dir = name.slice(0, name.length - GITIGNORE_FILENAME.length) matchers.push({ dir, filter: ignore().add(new TextDecoder().decode(content)) }) } return matchers } export function parsePackDependencies(specs: PackDependencySpecs): ParsedPackDependencies { return { npmDependencies: parseVersionedDeps(specs.npm ?? [], 'npm'), assetDependencies: parseVersionedDeps(specs.asset ?? [], 'asset'), skillDependencies: parseSkillDeps(specs.skill ?? []), } } export function packPolicyForType(type: AssetType | undefined): PackPolicy | undefined { if (!type) return undefined return { readPackageJsonDependencies: type === 'template', omitUnchangedInstalledFiles: type === 'template', } } export function packPolicyFromInstallMetadata( metadata: AssetInstallMetadata | undefined, ): PackPolicy { return { readPackageJsonDependencies: metadata?.readAssetDependenciesFromPackageJson ?? false, omitUnchangedInstalledFiles: metadata?.omitUnchangedInstalledFilesOnUpload ?? false, } } /** * Parse `name@range` specs (npm or asset deps) into a name→range record. The * range is optional and defaults to `*`. A leading `@` is treated as a scope * marker, so `@scope/pkg@^1.0.0` splits into `@scope/pkg` and `^1.0.0`. */ export function parseVersionedDeps(specs: string[], kind: 'npm' | 'asset'): Record { const out: Record = {} for (const spec of specs) { const at = spec.lastIndexOf('@') const hasRange = at > 0 const name = hasRange ? spec.slice(0, at) : spec const range = hasRange ? spec.slice(at + 1) : '*' if (!name || !range) { throw new Error(`Invalid ${kind} dependency "${spec}". Use name@range (e.g. three@^0.178.0).`) } if (name in out) { throw new Error(`Duplicate ${kind} dependency "${name}".`) } out[name] = range } return out } /** * Parse `label=source` specs into a label→source record. The source is passed * verbatim to `skills add` (a GitHub/git ref or a local path), so only the * first `=` is treated as the separator. */ export function parseSkillDeps(specs: string[]): Record { const out: Record = {} for (const spec of specs) { const eq = spec.indexOf('=') if (eq <= 0 || eq === spec.length - 1) { throw new Error( `Invalid skill dependency "${spec}". Use label=source ` + `(e.g. web-design=vercel-labs/agent-skills).`, ) } const label = spec.slice(0, eq) if (label in out) { throw new Error(`Duplicate skill dependency "${label}".`) } out[label] = spec.slice(eq + 1) } return out } function mergeAssetDependencies( fromPackageJson: AssetDependencies, explicit: Record, ): AssetDependencies { const merged = { ...fromPackageJson } for (const [name, range] of Object.entries(explicit)) { const existing = merged[name] if (existing !== undefined && assetDependencyRange(existing) !== range) { throw new Error( `Conflicting asset dependency "${name}": package.json has ${assetDependencyRange( existing, )}, --asset has ${range}`, ) } // A matching package.json entry wins: it may carry aliases the flag form cannot express. if (existing === undefined) merged[name] = range } return merged } function inferPackPolicy(files: Record): PackPolicy { const hasRootPackageJson = Boolean(files['package.json']) return { readPackageJsonDependencies: hasRootPackageJson, omitUnchangedInstalledFiles: hasRootPackageJson, } } // Drop each dependency file at its resolved location (one copy per file — duplicates ship as the // user's own content) so a template ships only its own (edited or new) files. Returns the // surviving entries plus the dependencies with freshly resolved aliases applied. async function withoutInstalledDependencyFiles( files: Record, assetDependencies: AssetDependencies, fetchFileManifest: FetchFileManifest | undefined, ): Promise<{ kept: Record assetDependencies: AssetDependencies skippedDependencies: string[] missingDependencyFiles: Array<{ name: string; path: string }> }> { const index = await dependencyFileIndex(assetDependencies, fetchFileManifest) const skippedDependencies = Object.keys(assetDependencies).filter( (name) => !index.fetched.has(name), ) if (index.files.length === 0) { return { kept: files, assetDependencies, skippedDependencies, missingDependencyFiles: [] } } const hashesByPath = new Map() const entryByNormalized = new Map() for (const [file, content] of Object.entries(files)) { const normalizedPath = normalizedZipPath(file) if (!normalizedPath || normalizedPath.endsWith('/')) continue hashesByPath.set(normalizedPath, md5(content)) entryByNormalized.set(normalizedPath, file) } const layout = resolveDependencyLayout(assetDependencies, index, hashesByPath) const omittedEntries = new Set( [...layout.omit].flatMap((omitted) => entryByNormalized.get(omitted) ?? []), ) const kept = Object.fromEntries( Object.entries(files).filter(([file]) => !omittedEntries.has(file)), ) return { kept, assetDependencies: layout.assetDependencies, skippedDependencies, missingDependencyFiles: layout.missing.map((file) => ({ name: file.name, path: file.canonicalPath, })), } } interface DependencyFile { name: string canonicalPath: string } interface DependencyFileEntry extends DependencyFile { /** md5 hex of the file's canonical content — its R2 etag, from the file index. */ hash: string } export interface DependencyFileIndex { files: DependencyFileEntry[] /** Dependencies whose manifest was fetched — only their aliases are re-derived. */ fetched: Set } function emptyDependencyFileIndex(): DependencyFileIndex { return { files: [], fetched: new Set() } } /** * The canonical content hash of every file belonging to a declared asset dependency, fetched from * the server (the version file indexes' etags). Returns an empty index when there's nothing to omit or no way * to fetch (offline / no client) — pack then keeps all files rather than guessing. */ async function dependencyFileIndex( assetDependencies: AssetDependencies, fetchFileManifest: FetchFileManifest | undefined, ): Promise { const index = emptyDependencyFileIndex() const entries = Object.entries(assetDependencies) if (!fetchFileManifest || entries.length === 0) return index const manifests = await Promise.all( entries.map(async ([name, value]) => { try { return { name, manifest: await fetchFileManifest(name, assetDependencyRange(value)) } } catch { // Can't reach the server for this dependency — keep its files rather than wrongly omitting them. return null } }), ) for (const result of manifests) { // An empty manifest is "unavailable", not "zero files": a real version always has files, but // the server returns {} for an unresolvable asset (deleted / private without access) or one // too large for manifest computation. Treating it as fetched would wipe the dep's aliases. if (!result || Object.keys(result.manifest).length === 0) continue index.fetched.add(result.name) for (const [file, hash] of Object.entries(result.manifest)) { index.files.push({ name: result.name, canonicalPath: file, hash }) } } return index } interface DependencyLayout { /** Project paths holding a dependency file — exactly one per dependency file found. */ omit: Set /** The declared dependencies with re-resolved aliases (fetched deps only; others untouched). */ assetDependencies: AssetDependencies /** Dependency files whose canonical bytes were found nowhere (locally modified or deleted). */ missing: DependencyFile[] } /** * Resolve where each dependency file lives in the project, by content hash. Exactly one project * copy is claimed per dependency file: the already-recorded alias target if its bytes still match * (stable across runs), else the canonical path, else the lexicographically first matching copy — * that copy is omitted from packs and recorded as the alias; further identical copies are the * user's own content. Stale aliases drop: a renamed file that was since modified ships as user * content and reinstall restores the canonical path. */ export function resolveDependencyLayout( declared: AssetDependencies, index: DependencyFileIndex, hashesByPath: Map, ): DependencyLayout { const pathsByHash = new Map() for (const [filePath, hash] of hashesByPath) { pathsByHash.set(hash, [...(pathsByHash.get(hash) ?? []), filePath]) } for (const paths of pathsByHash.values()) paths.sort() const omit = new Set() const missing: DependencyFile[] = [] const aliases: Record = {} for (const file of index.files) { const declaredValue = declared[file.name] const declaredAlias = declaredValue === undefined ? undefined : assetDependencyAlias(declaredValue)[file.canonicalPath] // A tombstone is a deliberate deletion: not missing, nothing to omit, and it survives as-is. if (declaredAlias === false) { aliases[file.name] = { ...(aliases[file.name] ?? {}), [file.canonicalPath]: false } continue } const candidates = (pathsByHash.get(file.hash) ?? []).filter((p) => !omit.has(p)) if (candidates.length === 0) { missing.push({ name: file.name, canonicalPath: file.canonicalPath }) continue } const target = declaredAlias !== undefined && candidates.includes(declaredAlias) ? declaredAlias : candidates.includes(file.canonicalPath) ? file.canonicalPath : candidates[0] omit.add(target) if (target !== file.canonicalPath) { aliases[file.name] = { ...(aliases[file.name] ?? {}), [file.canonicalPath]: target } } } const assetDependencies: AssetDependencies = {} for (const [name, value] of Object.entries(declared)) { assetDependencies[name] = index.fetched.has(name) ? assetDependencyValue(assetDependencyRange(value), aliases[name] ?? {}) : value } return { omit, assetDependencies, missing } } export interface AssetDependencySync { projectRoot: string /** Whether package.json was rewritten. */ changed: boolean assetDependencies: AssetDependencies /** Dependency files whose canonical bytes were found nowhere (locally modified or deleted). */ missing: Array<{ name: string; path: string }> /** Dependencies left untouched because their manifest could not be fetched. */ skipped: string[] } /** * Update the aliases in package.json `assetDependencies` to match where each dependency's files * currently live — the standalone form of what pack does before omitting (`market sync`). Renamed * files gain an alias, files back at their canonical path lose theirs, and dependencies whose * manifest cannot be fetched are reported as skipped rather than blindly rewritten. */ export async function syncAssetDependencies( cwd: string, fetchFileManifest: FetchFileManifest, ): Promise { const projectRoot = await findInstallRoot(cwd) const pkgPath = path.join(projectRoot, 'package.json') if (!(await maybeStat(pkgPath))?.isFile()) { throw new Error(`No package.json found at ${projectRoot}. Run sync inside a project.`) } const declared = packageJsonAssetDependencies(await fs.readFile(pkgPath, 'utf-8'), pkgPath) const names = Object.keys(declared) if (names.length === 0) { return { projectRoot, changed: false, assetDependencies: {}, missing: [], skipped: [] } } const index = await dependencyFileIndex(declared, fetchFileManifest) const walk = await walkWithGitignore(projectRoot) const hashesByPath = new Map() for (const relative of walk.kept) { hashesByPath.set( relative, md5(new Uint8Array(await fs.readFile(path.join(projectRoot, relative)))), ) } const layout = resolveDependencyLayout(declared, index, hashesByPath) const rewritten = await writeBackAssetDependencies(projectRoot, layout.assetDependencies) return { projectRoot, changed: rewritten !== null, assetDependencies: layout.assetDependencies, missing: layout.missing.map((file) => ({ name: file.name, path: file.canonicalPath })), skipped: names.filter((name) => !index.fetched.has(name)), } } /** * Persist detected aliases into the project's package.json (only for dependencies it already * declares — flag-only deps stay out of the file). Returns the serialized bytes when something * changed so the packed zip can carry the same content. */ async function writeBackAssetDependencies( dir: string, assetDependencies: AssetDependencies, ): Promise { const pkgPath = path.join(dir, 'package.json') const pkg = parsePackageJson(await fs.readFile(pkgPath, 'utf-8'), pkgPath) const declared = pkg.assetDependencies if (!declared) return null let changed = false for (const name of Object.keys(declared)) { const next = assetDependencies[name] if (next === undefined) continue if (JSON.stringify(declared[name]) === JSON.stringify(next)) continue declared[name] = next changed = true } if (!changed) return null const serialized = JSON.stringify(pkg, null, 2) + '\n' await fs.writeFile(pkgPath, serialized) return new TextEncoder().encode(serialized) } function normalizedZipPath(file: string): string | null { const zipPath = file.replace(/\\/g, '/') if ( zipPath.split('/').includes('..') || path.posix.isAbsolute(zipPath) || path.win32.isAbsolute(zipPath) ) { return null } return path.posix.normalize(zipPath) } async function resolveOneZipFile(cwd: string, zipFilter: string): Promise { const absolute = path.resolve(cwd, zipFilter) const stat = await maybeStat(absolute) if (stat?.isFile()) return assertZipFile(absolute) const files = await listFiles(cwd) const matches = files .filter((file) => matchesFilter(path.relative(cwd, file), zipFilter)) .filter(isZipFile) .sort() if (matches.length === 0) { throw new Error(`No .zip files matched "${zipFilter}"`) } if (matches.length > 1) { throw new Error(`File filter matched ${matches.length} zips; pack one asset at a time`) } return matches[0] } function assertZipFile(file: string): string { if (!isZipFile(file)) throw new Error(`Pack source must be a .zip: ${file}`) return file } function isZipFile(file: string): boolean { return /\.zip$/i.test(file) } async function maybeStat(file: string) { try { return await fs.stat(file) } catch (error) { if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null throw error } } async function listFiles(dir: string): Promise { const entries = await fs.readdir(dir, { withFileTypes: true }) const files: string[] = [] for (const entry of entries) { if (entry.name === 'node_modules' || entry.name === '.git') continue const fullPath = path.join(dir, entry.name) if (entry.isDirectory()) { files.push(...(await listFiles(fullPath))) } else if (entry.isFile()) { files.push(fullPath) } } return files } function matchesFilter(file: string, filter: string): boolean { const normalizedFile = file.split(path.sep).join('/') const normalizedFilter = filter.split(path.sep).join('/') const pattern = '^' + escapeRegExp(normalizedFilter) .replace(/\\\*\\\*/g, '.*') .replace(/\\\*/g, '[^/]*') + '$' return new RegExp(pattern).test(normalizedFile) } function escapeRegExp(s: string): string { return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') }