/**
 * cele2e build — build Docker images and package .netapp modules.
 *
 * Steps:
 *   1. Stage simulator inputs (website-sim static dist; npm-registry-sim
 *      @celilo/* tarballs). These COPY into the simulator images at
 *      docker build time, so they must be on disk first.
 *   2. Package each specified module directory as a .netapp file
 *   3. Build all Docker images from the @celilo/e2e docker/ directory
 *   4. Tag celilo-e2e/management as :vanilla — preserves the unbaked
 *      starting point for the install-sh regression test.
 *   5. Bake celilo into celilo-e2e/management:latest by running install.sh
 *      against the simulated celilo.computer / npm registry, then
 *      `docker commit`. The default management image every test uses is
 *      thus the actual install.sh outcome — drift in install.sh, the
 *      website static dist, or any @celilo/* workspace package fails
 *      build-infra loudly with the script's own output.
 *   6. Optionally save images to tarball (--save)
 */

import { execSync, spawnSync } from 'node:child_process';
import {
  closeSync,
  existsSync,
  mkdirSync,
  openSync,
  readFileSync,
  readSync,
  readdirSync,
  rmSync,
  writeFileSync,
} from 'node:fs';
import { basename, join, resolve } from 'node:path';
import { gunzipSync } from 'node:zlib';
import { stageAptRepo } from '../../scripts/stage-apt-repo';
import { stageLibsignal } from '../../scripts/stage-libsignal';
import { explainBuildFailure } from '../doctor';
import { isNetappCurrent } from '../netapp-staleness';
import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from '../registry-bundle';
import { findMonorepoRoot } from '../repo-root';
import { packNpmRegistryTarballs, stageWebsiteDist } from '../stage-simulator-inputs';

/**
 * Ceiling on a single `docker build`. The slowest image here is a few minutes
 * cold; 15 leaves generous headroom while bounding a wedged build, which
 * otherwise hangs the whole invocation with no signal at all.
 */
const PER_IMAGE_BUILD_TIMEOUT_MS = 15 * 60_000;

/**
 * Ceiling on a single `celilo module package` run. The slowest module takes
 * a couple of minutes; 10 leaves generous headroom. Without it a wedged
 * packaging step sat for two hours (2026-09-04, technitium: all samples in
 * kevent64, artifact truncated at 59 percent, no error, no exit).
 */
const PER_MODULE_PACKAGE_TIMEOUT_MS = 10 * 60_000;

/** Re-exported so this module's existing callers and its recurrence test keep their import path. */
export { findMonorepoRoot };

/** Return paths to all modules that have a manifest.yml. */
function discoverModuleDirs(repoRoot: string): string[] {
  const modulesDir = join(repoRoot, 'modules');
  return readdirSync(modulesDir)
    .map((name) => join(modulesDir, name))
    .filter((dir) => existsSync(join(dir, 'manifest.yml')));
}

const bold = '\x1b[1m';
const dim = '\x1b[2m';
const green = '\x1b[32m';
const red = '\x1b[31m';
const yellow = '\x1b[33m';
const reset = '\x1b[0m';

/**
 * Validate that a file is a complete gzip stream (full gunzip pass). This is
 * the check a .netapp or .tgz needs: a stream truncated mid-write fails here
 * with 'unexpected end of file' instead of surviving to the consumer, where
 * the identical zlib error reads as a truncated DOWNLOAD and sends the
 * debugging at the network while the artifact was born broken.
 *
 * Throws with an actionable message naming the file.
 */
export function assertGzipValid(filePath: string): void {
  try {
    gunzipSync(readFileSync(filePath));
    return;
  } catch (err) {
    const reason = err instanceof Error ? err.message : String(err);
    throw new Error(
      `${filePath} is not a valid gzip stream (${reason}). The artifact was written truncated or corrupted at rest; serving it would surface as 'zlib: unexpected end of file' on the consumer side, indistinguishable from a truncated download.`,
    );
  }
}

/**
 * Validate that a file is a real .deb (ar archive). A full dpkg parse is out
 * of scope; the magic bytes catch the truncated-at-write case, which is the
 * one that silently ships.
 */
function assertDebValid(filePath: string): void {
  const fd = openSync(filePath, 'r');
  try {
    const magic = Buffer.alloc(8);
    const read = readSync(fd, magic, 0, 8, 0);
    if (read < 8 || magic.toString('latin1') !== '!<arch>\n') {
      throw new Error(
        `${filePath} is not a valid ar archive (missing !<arch> magic). The .deb was written truncated or corrupted at rest.`,
      );
    }
  } finally {
    closeSync(fd);
  }
}

/**
 * Verify a produced or downloaded .netapp is a complete gzip stream. Throws
 * with an actionable message; callers fail the build rather than stage the
 * artifact for the sim registry to serve.
 */
export function verifyNetapp(filePath: string): void {
  if (!existsSync(filePath)) {
    throw new Error(`${filePath} was not produced — packaging step exited 0 but left no artifact`);
  }
  assertGzipValid(filePath);
}

/**
 * Verify every .netapp in the staging dir before declaring the stage done.
 * The sim registry binds this directory and serves whatever is in it, so a
 * corrupt file here becomes a 'zlib: unexpected end of file' on some consumer
 * that looks like a network fault. Catches stale artifacts from a previous
 * run too — the dir is not cleaned between builds.
 */
export function verifyStagedNetapps(netappsDir: string): void {
  if (!existsSync(netappsDir)) return;
  const netapps = readdirSync(netappsDir).filter((f) => f.endsWith('.netapp'));
  const failures: string[] = [];
  for (const netapp of netapps) {
    try {
      verifyNetapp(join(netappsDir, netapp));
    } catch (err) {
      failures.push(err instanceof Error ? err.message : String(err));
    }
  }
  if (failures.length > 0) {
    throw new Error(
      `${failures.length} of ${netapps.length} staged .netapp file(s) failed gzip integrity check:\n  ${failures.join('\n  ')}`,
    );
  }
}

interface BuildOptions {
  pkgDir: string;
  moduleDirs: string[];
  save: boolean;
  skipModules: boolean;
  /**
   * Bake `management:latest` from the PUBLISHED @celilo/cli on real npm
   * instead of the monorepo DEV cli served by the sim registry. Runs a
   * standalone container on the default docker network (real internet) —
   * the simulated topology has no route to npmjs.org.
   */
  published: boolean;
}

/**
 * Stage the build-time inputs that the simulator images COPY at docker-build
 * time: the website dist and the @celilo/* tarballs (via
 * src/stage-simulator-inputs.ts), then the apt-repo and libsignal debs.
 *
 * Both cache dirs are .gitignored — they're build outputs, not source. Without
 * this step, the simulator Dockerfiles fail at COPY (or worse, ship stale
 * tarballs that don't match the current workspace versions, which is what the
 * version-mismatch regression on 2026-05-06 caught).
 *
 * Pure no-op when the workspace doesn't have the cache targets —
 * keeps build-infra working in npm-installed @celilo/e2e, where these
 * staging dirs aren't relevant.
 */
async function stageSimulatorInputs(pkgDir: string): Promise<void> {
  // Try CWD first — when cele2e is installed globally, `pkgDir` is
  // somewhere like `~/.bun/install/global/node_modules/@celilo/e2e`,
  // which doesn't have a `modules/` sibling. Walking up from CWD
  // finds the user's actual monorepo. Falls back to `pkgDir` for the
  // in-monorepo dev case (cele2e symlinked or run via `bun run`).
  // Mirrors the same CWD-then-pkgDir pattern used for module
  // discovery further down.
  const repoRoot = findMonorepoRoot(process.cwd()) ?? findMonorepoRoot(pkgDir);
  if (!repoRoot) {
    // No monorepo (npm-installed @celilo/e2e) → fetch the same caches from
    // the public celilo sources instead of building them from workspace
    // source, so build-infra still works for a monorepo-free consumer (ce-i2i).
    await stageFromPublic(pkgDir);
    return;
  }

  const websiteSrc = join(repoRoot, 'modules', 'celilo-website', 'site');
  const packScript = join(pkgDir, 'scripts', 'pack-celilo-packages.ts');

  // Skip silently if neither dir exists — we're in an npm-installed
  // @celilo/e2e (no `modules/` source) or in a partial checkout.
  if (!existsSync(websiteSrc) && !existsSync(packScript)) return;

  console.log(`${bold}Staging simulator inputs...${reset}\n`);

  stageWebsiteDist(repoRoot, pkgDir);
  packNpmRegistryTarballs(repoRoot, pkgDir);

  // Build + stage the celilo/celilo-bootstrap .debs for apt-repo-sim.
  // Graceful no-op when packaging/ or nfpm is absent (npm-installed e2e).
  // Dockerfile.apt-repo-sim COPYs the staged .apt-repo-cache/pool at build
  // time, so this must run before buildDockerImages().
  stageAptRepo(repoRoot, pkgDir);

  // Build + stage the libsignal aarch64 native as a .deb, into the SAME
  // pool stageAptRepo just populated (so it must run after it — that function
  // recreates the pool empty). Compiled here on the host's own network rather
  // than inside the sealed e2e network; see stage-libsignal.ts.
  stageLibsignal(repoRoot, pkgDir);

  console.log('');
}

/**
 * The published @celilo/* packages the npm-registry-sim must serve so
 * install.sh's `bun add -g @celilo/cli @celilo/event-bus @celilo/e2e`
 * resolves its full transitive closure against the sim.
 *
 * This is the ONE place the install closure is a literal rather than derived:
 * build.ts runs consumer-side (inside the published @celilo/e2e tarball, with
 * NO monorepo present), so it cannot read the workspace package.json set at
 * runtime the way the monorepo packer does. `installClosure(readWorkspace())`
 * is the source of truth; this literal must match it, and
 * published-packages-in-sync.test.ts fails the instant they drift (that test is
 * why forgetting a package here can't recur — see WORKSPACE_PACKAGE_GRAPH.md).
 * Keep this sorted the same as installClosure()'s topo output for easy diffing.
 */
export const PUBLISHED_CELILO_PACKAGES = [
  '@celilo/capabilities',
  '@celilo/cli-display',
  '@celilo/core',
  '@celilo/event-bus',
  '@celilo/terraform-fake',
  '@celilo/e2e',
  '@celilo/cli',
] as const;

/**
 * npm-consumer counterpart to the monorepo staging above (ce-i2i). With no
 * `modules/` source tree to build from, fetch the sim-content caches from
 * the public celilo sources so `cele2e build-infra` still produces working
 * simulator images. Everything is pulled over the public internet, which
 * build-infra already requires (docker base images, the caddy apt repo, ACME):
 *
 *   .npm-registry-cache/   ← published @celilo/* tarballs (npm registry)
 *   .celilo-website-cache/ ← install.sh from https://celilo.computer (all the
 *                            bake step's `curl celilo.computer/install.sh` needs)
 *   .apt-repo-cache/pool/  ← real celilo + celilo-bootstrap .debs fetched from
 *                            apt.celilo.computer (only bootstrap-apt needs them).
 *
 * Sources are overridable via CELILO_E2E_NPM_REGISTRY / CELILO_E2E_WEBSITE_BASE
 * / CELILO_E2E_APT_BASE (used by the monorepo-free CI smoke test, ce-1pe).
 */
async function stageFromPublic(pkgDir: string): Promise<void> {
  console.log(`${bold}Staging simulator inputs (npm-consumer mode)...${reset}\n`);

  const registry = (process.env.CELILO_E2E_NPM_REGISTRY ?? 'https://registry.npmjs.org').replace(
    /\/+$/,
    '',
  );
  const websiteBase = (process.env.CELILO_E2E_WEBSITE_BASE ?? 'https://celilo.computer').replace(
    /\/+$/,
    '',
  );

  // (a) npm-registry-cache: the published @celilo/* tarballs.
  const npmCache = join(pkgDir, '.npm-registry-cache');
  rmSync(npmCache, { recursive: true, force: true });
  mkdirSync(npmCache, { recursive: true });
  process.stdout.write(`  ${'npm-registry (fetch)'.padEnd(28)} `);
  const t0 = Date.now();
  for (const pkg of PUBLISHED_CELILO_PACKAGES) {
    await fetchPublishedTarball(pkg, npmCache, registry);
  }
  console.log(
    `${green}✔${reset}  ${dim}${Math.round((Date.now() - t0) / 1000)}s, ${PUBLISHED_CELILO_PACKAGES.length} tarball(s)${reset}`,
  );

  // (b) celilo-website-cache: the bake step's `curl celilo.computer/install.sh`
  // only needs install.sh served, so fetch it (bootstrap.sh best-effort — it's
  // referenced by install.sh's help text but not needed for the bake path).
  const websiteCache = join(pkgDir, '.celilo-website-cache');
  rmSync(websiteCache, { recursive: true, force: true });
  mkdirSync(websiteCache, { recursive: true });
  process.stdout.write(`  ${'celilo-website (fetch)'.padEnd(28)} `);
  const t1 = Date.now();
  await fetchSiteFile(websiteBase, 'install.sh', websiteCache, { required: true });
  await fetchSiteFile(websiteBase, 'bootstrap.sh', websiteCache, { required: false });
  console.log(`${green}✔${reset}  ${dim}${Math.round((Date.now() - t1) / 1000)}s${reset}`);

  // (c) apt-repo-cache/pool: the real celilo + celilo-bootstrap .debs, fetched
  // from apt.celilo.computer so the bootstrap-apt test's `apt install
  // celilo-bootstrap` resolves. Only that test exercises this path; if the
  // fetch fails we leave the pool empty (build-infra still succeeds for
  // everyone) and bootstrap-apt fails loudly with the sim's "empty repo".
  const aptBase = (process.env.CELILO_E2E_APT_BASE ?? 'https://apt.celilo.computer').replace(
    /\/+$/,
    '',
  );
  const pool = join(pkgDir, '.apt-repo-cache', 'pool');
  rmSync(join(pkgDir, '.apt-repo-cache'), { recursive: true, force: true });
  mkdirSync(pool, { recursive: true });
  process.stdout.write(`  ${'apt-repo (fetch)'.padEnd(28)} `);
  const t2 = Date.now();
  try {
    const count = await fetchAptPool(aptBase, pool);
    console.log(
      `${green}✔${reset}  ${dim}${Math.round((Date.now() - t2) / 1000)}s, ${count} deb(s)${reset}`,
    );
  } catch (err) {
    console.log(
      `${yellow}empty${reset}  ${dim}(${err instanceof Error ? err.message : String(err)})${reset}`,
    );
  }

  console.log('');
}

/**
 * Fetch the celilo + celilo-bootstrap .debs from a real apt repo laid out like
 * apt.celilo.computer (`deb <base>/ stable main`) into `pool`. Parses the
 * per-arch Packages index for its `Filename: pool/...` entries and downloads
 * each referenced .deb (deduped across arches, so the arch-all bootstrap deb is
 * fetched once). Returns the number of debs staged; throws if none were found.
 */
export async function fetchAptPool(base: string, pool: string): Promise<number> {
  const filenames = new Set<string>();
  for (const arch of ['amd64', 'arm64']) {
    const dir = `${base}/dists/stable/main/binary-${arch}`;
    const packages = await fetchPackagesIndex(dir);
    for (const line of packages.split('\n')) {
      const m = line.match(/^Filename:\s*(\S+)/);
      if (m) filenames.add(m[1]);
    }
  }
  if (filenames.size === 0) {
    throw new Error(`no debs in ${base} stable/main — repo layout changed?`);
  }
  for (const filename of filenames) {
    const url = `${base}/${filename}`;
    const res = await fetch(url);
    if (!res.ok) throw new Error(`fetching ${url}: HTTP ${res.status}`);
    const debPath = join(pool, basename(filename));
    writeFileSync(debPath, Buffer.from(await res.arrayBuffer()));
    assertDebValid(debPath);
  }
  return filenames.size;
}

/** Fetch a Packages index from an apt binary-<arch> dir, uncompressed or .gz. */
async function fetchPackagesIndex(dir: string): Promise<string> {
  const plain = await fetch(`${dir}/Packages`);
  if (plain.ok) return await plain.text();
  const gz = await fetch(`${dir}/Packages.gz`);
  if (!gz.ok) throw new Error(`fetching ${dir}/Packages(.gz): HTTP ${gz.status}`);
  return gunzipSync(Buffer.from(await gz.arrayBuffer())).toString('utf8');
}

/**
 * Download a package's `latest` published tarball from the npm registry into
 * destDir. Uses the registry's own tarball filename so the npm-registry-sim
 * (which indexes tarballs by their embedded package.json) serves it verbatim.
 */
export async function fetchPublishedTarball(
  pkg: string,
  destDir: string,
  registry: string,
): Promise<void> {
  const packumentUrl = `${registry}/${pkg}`;
  const packumentRes = await fetch(packumentUrl);
  if (!packumentRes.ok) {
    throw new Error(
      `Fetching packument for ${pkg} (${packumentUrl}): HTTP ${packumentRes.status}. ` +
        `Is ${pkg} published to ${registry}?`,
    );
  }
  const packument = (await packumentRes.json()) as {
    'dist-tags'?: Record<string, string>;
    versions?: Record<string, { dist?: { tarball?: string } }>;
  };
  const latest = packument['dist-tags']?.latest;
  const tarballUrl = latest ? packument.versions?.[latest]?.dist?.tarball : undefined;
  if (!latest || !tarballUrl) {
    throw new Error(`${pkg}: registry ${registry} has no dist-tags.latest tarball`);
  }
  const tgzRes = await fetch(tarballUrl);
  if (!tgzRes.ok) {
    throw new Error(`Fetching tarball for ${pkg}@${latest} (${tarballUrl}): HTTP ${tgzRes.status}`);
  }
  const filename =
    tarballUrl.split('/').pop() ?? `${pkg.replace('@', '').replace('/', '-')}-${latest}.tgz`;
  const tgzPath = join(destDir, filename);
  writeFileSync(tgzPath, Buffer.from(await tgzRes.arrayBuffer()));
  // A truncated tarball in .npm-registry-cache fails the consumer's bun add
  // with a zlib error that reads like a network fault. Check here where the
  // cause (a bad fetch) is still visible.
  assertGzipValid(tgzPath);
}

/**
 * Download a single file from the published site into destDir. Throws on a
 * missing required file; silently skips an absent optional one.
 */
export async function fetchSiteFile(
  base: string,
  name: string,
  destDir: string,
  opts: { required: boolean },
): Promise<void> {
  const url = `${base}/${name}`;
  let res: Response;
  try {
    res = await fetch(url);
  } catch (err) {
    if (opts.required) throw err;
    return;
  }
  if (!res.ok) {
    if (opts.required) throw new Error(`Fetching ${url}: HTTP ${res.status}`);
    return;
  }
  writeFileSync(join(destDir, name), Buffer.from(await res.arrayBuffer()));
}

/**
 * Package one module directory into `netappsDir` as `<name>.netapp`, skipping
 * the work entirely when the staged netapp is already current over the source
 * (celilo#1258). Exported for its recurrence test.
 */
export function packageNetapp(moduleDir: string, netappsDir: string): void {
  const absDir = resolve(moduleDir);
  if (!existsSync(absDir)) {
    console.error(`  ${red}skip${reset}  ${moduleDir}  ${dim}(not found)${reset}`);
    return;
  }

  const name = basename(absDir);
  const out = join(netappsDir, `${name}.netapp`);

  // Skip a module whose staged .netapp is newer than every source file
  // (celilo#1258): repackaging it again would reproduce the same bytes, and
  // doing that for all 37 modules cost about 5 minutes on every build-infra
  // run. A source edit flips the newest mtime and the module repackages.
  if (isNetappCurrent(out, absDir)) {
    console.log(`  ${String(name).padEnd(20)} ${dim}· current, skipped${reset}`);
    return;
  }

  process.stdout.write(`  ${String(name).padEnd(20)} `);
  const start = Date.now();

  // Find celilo CLI by locating the monorepo root from the module's directory.
  const repoRoot = findMonorepoRoot(absDir);
  const celiloWrapper = repoRoot ? join(repoRoot, 'celilo') : null;
  const celiloTs = repoRoot ? join(repoRoot, 'apps/celilo/src/cli/index.ts') : null;

  let result: ReturnType<typeof spawnSync>;
  if (celiloWrapper && existsSync(celiloWrapper)) {
    result = spawnSync(celiloWrapper, ['package', absDir, '--output', out], {
      stdio: 'pipe',
      timeout: PER_MODULE_PACKAGE_TIMEOUT_MS,
    });
  } else if (celiloTs && existsSync(celiloTs)) {
    result = spawnSync('bun', ['run', celiloTs, 'package', absDir, '--output', out], {
      stdio: 'pipe',
      timeout: PER_MODULE_PACKAGE_TIMEOUT_MS,
    });
  } else {
    console.error(`  ${red}skip${reset}  ${name}  ${dim}(celilo CLI not found)${reset}`);
    return;
  }

  const elapsed = Math.round((Date.now() - start) / 1000);
  if (result.error && (result.error as NodeJS.ErrnoException).code === 'ETIMEDOUT') {
    console.log(`${red}✗ TIMED OUT${reset}  ${dim}${elapsed}s${reset}`);
    console.error(
      `\n${red}Packaging ${name} exceeded ${Math.round(PER_MODULE_PACKAGE_TIMEOUT_MS / 60_000)} minutes and was killed.${reset}`,
    );
    console.error(
      `${dim}The stall is usually transient (2026-09-04: technitium wedged two hours, artifact truncated mid-write). Delete ${out} and re-run \`cele2e build-infra\`.${reset}`,
    );
    process.exit(1);
  }
  if (result.status !== 0) {
    console.log(`${red}✗${reset}  ${dim}${result.stderr?.toString().trim()}${reset}`);
    // A failed package must fail the build, not silently shrink the module
    // set: the sim registry binds netapps/ as-is, so the omission surfaces
    // later as a confusing deploy failure instead of a nameable one.
    process.exit(1);
  }

  // Verify what we just wrote before declaring the step done. A wedge in the
  // packaging stream leaves a truncated file that still exits 0 (celilo#1257).
  try {
    verifyNetapp(out);
  } catch (err) {
    console.log(`${red}✗ CORRUPT${reset}  ${dim}${elapsed}s${reset}`);
    console.error(`\n${red}${err instanceof Error ? err.message : String(err)}${reset}`);
    process.exit(1);
  }

  const size = (() => {
    try {
      return execSync(`du -h "${out}"`).toString().split('\t')[0];
    } catch {
      return '?';
    }
  })();

  console.log(`${green}✔${reset}  ${dim}${elapsed}s  ${size}${reset}`);
}

/**
 * Consumer (npm-installed) staging: fetch standard-module .netapps from the
 * public celilo registry into `netappsDir`. In the monorepo we package these
 * from source (packageNetapp); an npm consumer has no `modules/` source, so
 * getRegistryVolumes() binds `./netapps` into the sim registry and we fill it
 * here at build-infra time (which has network — the test RUN stays hermetic
 * against the local sim). No version-pinning: the capability system governs
 * compatibility at import time (validateCapabilityAccess), surfacing a
 * mismatch as an actionable import error rather than a brittle lockfile.
 */
export async function stageNetappsFromRegistry(netappsDir: string): Promise<void> {
  const base = (process.env.CELILO_REGISTRY_URL ?? 'https://celilo.computer/registry').replace(
    /\/+$/,
    '',
  );
  console.log(`${bold}Fetching standard-module netapps from registry...${reset}`);
  console.log(`  ${dim}${base}${reset}\n`);

  // ponytail: single page (per_page=100). Paginate if >100 modules ever ship.
  const listUrl = `${base}/api/v1/modules?per_page=100`;
  const listResp = await fetch(listUrl, { signal: AbortSignal.timeout(30_000) });
  if (!listResp.ok) {
    console.error(`${red}Registry list failed: HTTP ${listResp.status} ${listUrl}${reset}`);
    process.exit(1);
  }
  const { modules } = (await listResp.json()) as {
    modules: Array<{ name: string; max_version: string }>;
  };
  if (!modules || modules.length === 0) {
    console.error(`${red}Registry returned no modules — nothing to stage${reset}`);
    process.exit(1);
  }

  for (const { name, max_version } of modules) {
    process.stdout.write(`  ${name.padEnd(20)} `);
    const t0 = Date.now();
    const dlUrl = `${base}/api/v1/modules/${encodeURIComponent(name)}/${encodeURIComponent(max_version)}/download`;
    const resp = await fetch(dlUrl, { signal: AbortSignal.timeout(60_000) });
    if (!resp.ok) {
      console.log(`${red}✗${reset}`);
      console.error(`${red}  download failed: HTTP ${resp.status} ${dlUrl}${reset}`);
      process.exit(1);
    }
    const bytes = new Uint8Array(await resp.arrayBuffer());
    const netappPath = join(netappsDir, `${name}.netapp`);
    writeFileSync(netappPath, bytes);
    try {
      verifyNetapp(netappPath);
    } catch (err) {
      console.log(`${red}✗ CORRUPT${reset}`);
      console.error(`${red}  ${err instanceof Error ? err.message : String(err)}${reset}`);
      process.exit(1);
    }
    console.log(
      `${green}✔${reset}  ${dim}${max_version}  ${Math.round((Date.now() - t0) / 1000)}s  ${(bytes.length / 1024).toFixed(0)}KB${reset}`,
    );
  }
  console.log('');
}

function buildDockerImages(pkgDir: string): void {
  // Refresh the bundled registry-server source before docker build —
  // Dockerfile.registry copies from <pkgDir>/registry-server, which is
  // .gitignored and recreated from the canonical source each build.
  ensureRegistryServerBundle(pkgDir);
  ensureTerraformFakeBundle(pkgDir);

  const dockerDir = join(pkgDir, 'docker');
  if (!existsSync(dockerDir)) {
    console.error(`${red}Docker directory not found: ${dockerDir}${reset}`);
    process.exit(1);
  }

  const dockerfiles = readdirSync(dockerDir)
    .filter((f) => f.startsWith('Dockerfile.'))
    .sort();

  if (dockerfiles.length === 0) {
    console.log(`${dim}No Dockerfiles found in ${dockerDir}${reset}`);
    return;
  }

  const total = dockerfiles.length;
  const buildStart = Date.now();

  // Per-Dockerfile build-context overrides. Default context is the e2e
  // package root. Dockerfile.registry uses the same default — the
  // registry-server source is bundled inside the package (see
  // ensureRegistryServerBundle above) so Dockerfile.registry's
  // `COPY registry-server/...` resolves to <pkgDir>/registry-server/
  // both in the monorepo (regenerated from sibling) and when
  // npm-installed (shipped in tarball via the package's `files:` list).
  const contextOverrides: Record<string, string> = {};

  for (let i = 0; i < total; i++) {
    const file = dockerfiles[i];
    const name = file.replace('Dockerfile.', '');
    const tag = `celilo-e2e/${name}`;
    const context = contextOverrides[file] ?? '.';

    process.stdout.write(`  [${i + 1}/${total}] ${String(name).padEnd(25)} `);

    const start = Date.now();
    const result = spawnSync('docker', ['build', '-t', tag, '-f', join(dockerDir, file), context], {
      cwd: pkgDir,
      stdio: 'pipe',
      timeout: PER_IMAGE_BUILD_TIMEOUT_MS,
    });
    const elapsed = Math.round((Date.now() - start) / 1000);

    // A wedged `docker build` used to hang here forever: spawnSync blocks the
    // event loop, so even the run-lock heartbeat stops — the holder looks alive
    // (its PID is) while beating nothing, and the only way anyone noticed was
    // comparing beatAt to wall-clock by hand. Twice in one evening, ~20 and ~32
    // minutes. A bounded wait turns that into an immediate, nameable failure.
    if (result.error && (result.error as NodeJS.ErrnoException).code === 'ETIMEDOUT') {
      console.log(`${red}✗ TIMED OUT${reset}  ${dim}${elapsed}s${reset}`);
      console.error(
        `\n${red}Building ${name} exceeded ${Math.round(PER_IMAGE_BUILD_TIMEOUT_MS / 60_000)} minutes and was killed.${reset}`,
      );
      console.error(
        `${dim}Docker itself is usually still responsive. Reproduce it alone with:${reset}`,
      );
      console.error(`${dim}  docker build -f docker/${file} -t ${tag} .${reset}`);
      console.error(
        `${dim}If that succeeds, the stall was transient — re-run \`cele2e build-infra\`.${reset}\n`,
      );
      process.exit(1);
    }

    if (result.status !== 0) {
      console.log(`${red}✗${reset}  ${dim}${elapsed}s${reset}`);
      console.error(result.stderr?.toString());
      console.error(explainBuildFailure(result.stderr?.toString() ?? ''));
      process.exit(1);
    }

    console.log(`${green}✔${reset}  ${dim}${elapsed}s${reset}`);
  }

  const totalElapsed = Math.round((Date.now() - buildStart) / 1000);
  console.log(`\n${green}All ${total} images built in ${totalElapsed}s${reset}`);
}

/**
 * Tag the just-built management image as `:vanilla` before the bake step
 * rewrites `:latest`. The `:vanilla` tag stays addressable so the
 * install-sh regression test (e2e/tests/install-sh.test.ts) can still
 * exercise an unbaked starting point.
 */
function tagVanillaManagement(): void {
  process.stdout.write(`  ${'management:vanilla (tag)'.padEnd(28)} `);
  const result = spawnSync(
    'docker',
    ['tag', 'celilo-e2e/management', 'celilo-e2e/management:vanilla'],
    { stdio: 'pipe' },
  );
  if (result.status !== 0) {
    console.log(`${red}✗${reset}`);
    console.error(result.stderr?.toString());
    process.exit(1);
  }
  console.log(`${green}✔${reset}`);
}

/**
 * Run the install.sh-via-vanilla-management bake step, producing the
 * post-install-sh `celilo-e2e/management:latest` image. Delegates to the
 * `bin/e2e-bake-management` orchestrator (which uses the @celilo/e2e
 * `network()` builder to spin up shared infra + minimal per-test stack,
 * runs install.sh inside vanilla management, verifies celilo --version,
 * installs the source-mount shim, and `docker commit`s).
 *
 * Failure here means install.sh is broken — by design, this fails
 * build-infra so the operator can't accidentally proceed with a
 * non-functional default management image.
 */
export function bakeManagement(pkgDir: string, published: boolean): void {
  const bakeScript = join(pkgDir, 'bin', 'e2e-bake-management');
  if (!existsSync(bakeScript)) {
    console.error(
      `${red}Bake script missing: ${bakeScript}${reset}\n` +
        `${dim}This is required to produce celilo-e2e/management:latest.${reset}`,
    );
    process.exit(1);
  }

  console.log(
    published
      ? `${bold}Baking PUBLISHED @celilo/cli into management:latest from real npm...${reset}\n`
      : `${bold}Baking celilo into management:latest via install.sh...${reset}\n`,
  );
  const start = Date.now();
  const bakeArgs = ['run', bakeScript, ...(published ? ['--published'] : [])];
  const result = spawnSync('bun', bakeArgs, { stdio: 'inherit' });
  const elapsed = Math.round((Date.now() - start) / 1000);

  if (result.status !== 0) {
    console.log(`${red}✗${reset}  ${dim}${elapsed}s${reset}`);
    // status is null when the child died to a signal — that is a genuine
    // failure, not a refusal, so it falls through to the exit-1 report.
    reportBakeChildFailure(result.status ?? 1, elapsed);
  }

  console.log(`\n${green}Baked in ${elapsed}s${reset}\n`);
}

/**
 * Report a non-zero bake-child exit and terminate. Exit 3 is the live-stack
 * refusal convention (celilo#1297 guard; the run runner keys on the same code
 * in its `refused` field): the bake child's startup cleanup found a live
 * celilo-e2e-* stack or a foreign run lock and refused before the bake did
 * any work. Reporting that as a bake failure named three causes — install.sh
 * rot, registry tarballs, website sim — for a step that never executed
 * (celilo#1302: 9 of ~111 smoke runs, every refusal diagnosed as install.sh
 * rot). A refusal is not a bake failure, so it gets its own message and exits
 * 3 so callers can tell the two apart.
 */
export function reportBakeChildFailure(status: number, elapsed: number): never {
  if (status === 3) {
    console.error(
      `\n${red}Bake step did not run — the startup cleanup refused: a live e2e stack is in the way.${reset}`,
    );
    console.error(
      `${dim}A refusal is not a bake failure; install.sh was never exercised. Clear the stack with \`cele2e down\` (or free the foreign run lock) and re-run \`cele2e build-infra\`.${reset}`,
    );
    process.exit(3);
  }
  console.error(`\n${red}Bake step failed after ${elapsed}s${reset}`);
  console.error(
    `${dim}Likely causes: install.sh regressed, npm-registry-sim doesn't have the expected @celilo/* tarballs, or celilo-website-sim isn't serving install.sh. Run \`cele2e run install-sh\` for a focused reproducer with cleaner output.${reset}`,
  );
  process.exit(1);
}

/**
 * Pre-seed heavy application images (authentik server, postgres, redis) into
 * the app-zone preload cache. Without this the `docker-image-cache` is empty on
 * a fresh checkout / the builder, so every app-module deploy pulls GB through
 * the simulated internet — slow enough to time the authentik deploy out
 * (exit 124) and take down forgejo-deploy + full-stack-pipeline (ISS-0154).
 * Best-effort: `e2e-cache-images` is per-image resilient and the preload
 * service skips a missing tarball, so a transient registry hiccup degrades to a
 * slow pull rather than failing build-infra.
 */
function cacheAppImages(pkgDir: string): void {
  const cacheScript = join(pkgDir, 'bin', 'e2e-cache-images');
  if (!existsSync(cacheScript)) return;

  console.log(`${bold}Pre-seeding app-zone image cache...${reset}\n`);
  const result = spawnSync('bash', [cacheScript], { stdio: 'inherit' });
  if (result.status !== 0) {
    console.error(
      `${yellow}⚠ image cache step failed — app-zone deploys may pull over the sim${reset}`,
    );
  }
  console.log('');
}

function saveImages(pkgDir: string): void {
  const cacheDir = join(pkgDir, '.docker-cache');
  mkdirSync(cacheDir, { recursive: true });
  const tarball = join(cacheDir, 'celilo-e2e-images.tar');

  console.log(`\n${bold}Saving images to tarball...${reset}`);

  const images = execSync(
    'docker images --filter "reference=celilo-e2e/*" --format "{{.Repository}}:{{.Tag}}"',
  )
    .toString()
    .trim()
    .split('\n')
    .filter(Boolean)
    .sort();

  if (images.length === 0) {
    console.error(`${red}No celilo-e2e/* images found to save${reset}`);
    return;
  }

  const result = spawnSync('docker', ['save', ...images, '-o', tarball], { stdio: 'inherit' });
  if (result.status !== 0) process.exit(1);

  const size = (() => {
    try {
      return execSync(`du -h "${tarball}"`).toString().split('\t')[0];
    } catch {
      return '?';
    }
  })();

  console.log(`${green}Saved to ${tarball} (${size})${reset}`);
  console.log('\nTo restore after colima restart:\n  cele2e load');
}

/**
 * Collect the untagged images this build (and every build before it) left
 * behind.
 *
 * `docker image prune -f` — no `-a`, no `system prune` — removes only images
 * that carry no tag AND no container. The tagged base images every Dockerfile
 * starts `FROM` are therefore untouched, which is the whole distinction: the
 * warning in this repo's operating notes is about `-a` and `system prune`,
 * which DO delete `ubuntu:22.04` and cost a full 27-image rebuild that then
 * gets misread as a network failure (`failed to solve: … TLS handshake
 * timeout`). The bare form is the opposite — it is what stops the pile that
 * drives people to reach for the destructive one.
 *
 * Measured 2026-09-05 before this existed: 3178 dangling images, 25.8 GB, 97%
 * of the local image store, mostly management images superseded by a bake.
 */
function pruneSupersededImages(): void {
  process.stdout.write(`${bold}Removing superseded (untagged) images...${reset} `);
  try {
    const out = execSync('docker image prune -f', { encoding: 'utf-8', timeout: 300_000 });
    const reclaimed = out.match(/Total reclaimed space:\s*(.+)/)?.[1]?.trim() ?? '0B';
    console.log(`${green}✔${reset} ${dim}${reclaimed} reclaimed${reset}`);
  } catch {
    // Never fail a build over housekeeping.
    console.log(`${yellow}⚠ skipped${reset}`);
  }
}

export async function runBuild(options: BuildOptions): Promise<void> {
  const { pkgDir, moduleDirs, save, skipModules, published } = options;
  const netappsDir = join(pkgDir, 'netapps');
  mkdirSync(netappsDir, { recursive: true });

  // Stage simulator inputs first — Dockerfiles for celilo-website-sim
  // and npm-registry-sim COPY from the staged dirs at docker-build time,
  // so they have to exist (and reflect current workspace state) before
  // buildDockerImages() runs.
  await stageSimulatorInputs(pkgDir);

  if (!skipModules) {
    // Explicit dirs take precedence; fall back to auto-discovering all
    // modules. Try CWD first so a globally-installed cele2e finds the
    // user's monorepo (the package's own dir is in
    // ~/.bun/install/global/... and won't have a `modules/` sibling).
    // Fall back to the package dir for the in-monorepo dev case.
    const dirs =
      moduleDirs.length > 0
        ? moduleDirs
        : (() => {
            const root = findMonorepoRoot(process.cwd()) ?? findMonorepoRoot(pkgDir);
            return root ? discoverModuleDirs(root) : [];
          })();

    if (dirs.length > 0) {
      console.log(`${bold}Packaging modules...${reset}\n`);
      for (const dir of dirs) {
        packageNetapp(dir, netappsDir);
      }
      console.log('');
    } else {
      // npm consumer: no `modules/` source to package from — fetch the
      // standard-module netapps from the registry so the sim registry
      // (which binds ./netapps) has something to serve. Consumers need
      // zero vendored module binaries.
      await stageNetappsFromRegistry(netappsDir);
    }
  }

  // Final gate before anything serves these files: the sim registry binds
  // netapps/ verbatim, so every artifact in it (fresh or stale from a prior
  // run) must be a complete gzip stream. A truncated .netapp served to a
  // consumer reports 'zlib: unexpected end of file', byte-identical to a
  // truncated download, which sends the debugging at the network (celilo#1257).
  try {
    verifyStagedNetapps(netappsDir);
  } catch (err) {
    console.error(`\n${red}${err instanceof Error ? err.message : String(err)}${reset}`);
    console.error(
      `${dim}Delete the corrupt file(s) and re-run \`cele2e build-infra\`. If packaging reproduced the corruption, the stall was likely transient — re-run before deeper debugging.${reset}`,
    );
    process.exit(1);
  }

  console.log(`${bold}Building E2E Docker images...${reset}\n`);
  buildDockerImages(pkgDir);

  // After docker build, the management image at celilo-e2e/management
  // is vanilla (bun + unzip + bunfig, no celilo). Tag it as :vanilla
  // before bake rewrites :latest, so the install-sh regression test
  // still has an unbaked starting point.
  tagVanillaManagement();
  console.log('');

  // Bake celilo into :latest by running install.sh through the
  // simulated celilo.computer / npm registry. This is what makes the
  // default management image every test uses an actual product of
  // install.sh, rather than a parallel `bun add -g` pinning.
  bakeManagement(pkgDir, published);

  // Pre-seed heavy app images so app-zone deploys load them from the cache
  // instead of pulling GB through the sim (ISS-0154).
  cacheAppImages(pkgDir);

  if (save) {
    saveImages(pkgDir);
  }

  // Last, so it also collects the image the bake just superseded.
  pruneSupersededImages();
}