/** * Bootstrap mode — optional behavior for the e2e stand-in registry. * * When BOOTSTRAP_MODULES_DIR is set, the server also serves modules whose * source lives in that directory. Each immediate subdirectory containing a * manifest.yml is treated as a module; its manifest's `id` and `version` * become the sparse index entry. On download, the directory is packaged * into a .netapp tarball on the fly (cached on disk). * * This preserves the existing packages/e2e/docker/Dockerfile.registry * behavior where tests can import any module from the repo without having * to explicitly publish each dependency first. * * In production, BOOTSTRAP_MODULES_DIR is unset and this whole layer is * inert — storage.ts is the only code that touches disk. */ import { createHash } from 'node:crypto'; import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { basename, dirname, join, relative } from 'node:path'; import type { IndexEntry } from './storage'; export interface BootstrapEntry { name: string; version: string; /** Source directory to package on demand (source mode). */ sourceDir?: string; /** Pre-built .netapp to serve directly (upload mode). */ netappPath?: string; /** * Manifest's `description` field, when present. Used by the search * endpoint to populate the module-list payload — see * apps/celilo/designs/REGISTRY_BROWSE_UI.md (Phase 2 step 0). */ description?: string; /** Manifest's `icon` field, when present. Served like `description`. */ icon?: string; } /** * Scan a directory for modules. Each subdirectory is a module iff it contains * a manifest.yml. Manifest parsing is line-regex (not full YAML) so we don't * need a YAML dependency at runtime — this matches packages/e2e/src/registry-server.ts. */ export function scanBootstrapDir(bootstrapDir: string): Map { const entries = new Map(); if (!existsSync(bootstrapDir)) return entries; for (const entry of readdirSync(bootstrapDir, { withFileTypes: true })) { if (!entry.isDirectory()) continue; const manifestPath = join(bootstrapDir, entry.name, 'manifest.yml'); if (!existsSync(manifestPath)) continue; try { const yaml = readFileSync(manifestPath, 'utf-8'); const idMatch = yaml.match(/^id:\s*['"]?([^\s'"#]+)['"]?/m); const versionMatch = yaml.match(/^version:\s*['"]?([^\s'"#]+)['"]?/m); // Description is the rest-of-line up to (but not including) the // newline. The manifest schema allows multi-line scalars, but // existing modules use the single-line form so the line-regex is // sufficient. We strip surrounding quotes if present. const descriptionMatch = yaml.match(/^description:\s*(.+?)\s*$/m); const name = idMatch?.[1]?.trim() ?? entry.name; const version = versionMatch?.[1]?.trim() ?? '0.0.0'; const description = descriptionMatch?.[1]?.trim().replace(/^['"]|['"]$/g, ''); const iconMatch = yaml.match(/^icon:\s*(.+?)\s*$/m); const icon = iconMatch?.[1]?.trim().replace(/^['"]|['"]$/g, ''); entries.set(name, { name, version, sourceDir: join(bootstrapDir, entry.name), description, icon, }); } catch { // skip malformed manifests — matches e2e server behavior } } return entries; } /** * Scan a directory for pre-built .netapp files. Filename (minus suffix) is * the module name. Version defaults to the filename's version suffix or to * 0.0.0+1 — real version data is in the archive's manifest but parsing that * at startup is overkill for the e2e bootstrap use case. * * Used by the shared-infra stand-in: the e2e harness's publishModule helper * copies .netapps into BOOTSTRAP_UPLOADS_DIR with `docker cp`, and the * server picks them up without needing the full HTTP publish protocol. */ export function scanUploadsDir(uploadsDir: string): Map { const entries = new Map(); if (!existsSync(uploadsDir)) return entries; for (const file of readdirSync(uploadsDir)) { if (!file.endsWith('.netapp')) continue; const name = file.slice(0, -7); entries.set(name, { name, version: '0.0.0', netappPath: join(uploadsDir, file), }); } return entries; } /** Render a bootstrap entry as an IndexEntry suitable for the sparse index. */ export function bootstrapIndexEntry(entry: BootstrapEntry): IndexEntry { const vers = entry.version.includes('+') ? entry.version : `${entry.version}+1`; return { name: entry.name, vers, deps: [], cksum: 'bootstrap', yanked: false, }; } /** * Walk a directory, skipping hidden entries, *.test.ts files, the * module's e2e/ tree (tests + their deps don't ship with the module), * and most of node_modules. * * `scripts/node_modules` (the module's hook runtime) is bundled IN FULL * (ISS-0046): hook scripts run on the target, where the package registry can * be unreachable, so the `.netapp` must carry @celilo/capabilities AND its * third-party deps (tldts, drizzle-orm, …). Any OTHER node_modules ships only * `@celilo/capabilities` — the framework SDK the module was authored against. * * This is the structural twin of the canonical `includeNodeModulesPath` rule in * apps/celilo/src/module/packaging/package-rules.ts. We can't import that at * runtime (this server ships in a standalone Docker image with no @celilo deps), * so the two are held in lockstep by bootstrap-packaging.test.ts, which asserts * `walkDir`'s output conforms to the canonical rule. `walkDir` is exported for * that test. */ export function walkDir(dir: string, base = dir, inNodeModules = false): string[] { const results: string[] = []; for (const entry of readdirSync(dir, { withFileTypes: true })) { if (entry.name.startsWith('.')) continue; if (entry.name.endsWith('.test.ts')) continue; // The module's own tsconfig.json is dev-only (CI typechecks hooks with it); // build.ts drops it too. A dependency's tsconfig.json inside node_modules is // that package's published content and stays — build.ts keeps those as well. if (entry.name === 'tsconfig.json' && !inNodeModules) continue; const fullPath = join(dir, entry.name); // Module's e2e/ directory at the root is tests + their devDependencies. // Detected by being a sibling of manifest.yml. if (entry.name === 'e2e' && existsSync(join(dir, 'manifest.yml'))) { continue; } if (entry.name === 'node_modules') { if (basename(dir) === 'scripts') { // The hook-script runtime closure — bundle it whole (incl. nested deps). // walkAll keeps the hidden-entry / *.test.ts skips, so .bin shims drop. results.push(...walkAll(fullPath, base)); } else { // Other node_modules: collect only @celilo/capabilities. const capDir = join(fullPath, '@celilo', 'capabilities'); if (existsSync(capDir)) { results.push(...walkDir(capDir, base, true)); } } continue; } if (entry.isDirectory()) { results.push(...walkDir(fullPath, base, inNodeModules)); } else { results.push(relative(base, fullPath)); } } return results; } /** * Collect every file under `dir` (recursing into nested node_modules too), * skipping only hidden entries (`.bin` symlink shims) and `*.test.ts`. Used to * bundle a module's full `scripts/node_modules` runtime closure. */ function walkAll(dir: string, base: string): string[] { const results: string[] = []; for (const entry of readdirSync(dir, { withFileTypes: true })) { if (entry.name.startsWith('.')) continue; if (entry.name.endsWith('.test.ts')) continue; const fullPath = join(dir, entry.name); if (entry.isDirectory()) { results.push(...walkAll(fullPath, base)); } else { results.push(relative(base, fullPath)); } } return results; } /** * Package a module directory into a .netapp tarball. Cached by module name * in cacheDir; cache is keyed by name only (assume the source is immutable * for the lifetime of the server — fine for e2e). */ export async function packageBootstrapModule( entry: BootstrapEntry, cacheDir: string, ): Promise { // Upload-mode entries are already .netapp files — serve as-is. if (entry.netappPath) return entry.netappPath; const sourceDir = entry.sourceDir; if (!sourceDir) { throw new Error(`bootstrap entry ${entry.name} has neither sourceDir nor netappPath`); } mkdirSync(cacheDir, { recursive: true }); const cachePath = join(cacheDir, `${entry.name}.netapp`); if (existsSync(cachePath)) return cachePath; const filePaths = walkDir(sourceDir); const fileHashes: Record = {}; for (const relPath of filePaths) { const data = readFileSync(join(sourceDir, relPath)); fileHashes[relPath] = createHash('sha256').update(data).digest('hex'); } const checksumsJson = JSON.stringify( { version: '1', generated: new Date().toISOString(), files: fileHashes }, null, 2, ); // Source dir may be read-only (volume-mounted) — stage in /tmp first. const stagingDir = join('/tmp', `celilo-bootstrap-${entry.name}-${Date.now()}`); mkdirSync(stagingDir, { recursive: true }); try { for (const relPath of filePaths) { mkdirSync(dirname(join(stagingDir, relPath)), { recursive: true }); writeFileSync(join(stagingDir, relPath), readFileSync(join(sourceDir, relPath))); } writeFileSync(join(stagingDir, 'checksums.json'), checksumsJson); const proc = Bun.spawn(['tar', '-czf', cachePath, '-C', stagingDir, '.'], { stderr: 'inherit', }); const exitCode = await proc.exited; if (exitCode !== 0) throw new Error(`tar exited with code ${exitCode}`); } finally { try { Bun.spawnSync(['rm', '-rf', stagingDir]); } catch { // best-effort cleanup } } return cachePath; }