#!/usr/bin/env bun
/**
 * Bake celilo into the management image, then `docker commit` the result
 * onto the `celilo-e2e/management:latest` tag. Two modes:
 *
 *   (default)     Run install.sh against the simulated celilo.computer +
 *                 npm registry, installing the monorepo DEV cli exactly the
 *                 way users do. Exercises install.sh end-to-end.
 *
 *   --published   Install the PUBLISHED @celilo/cli from REAL npm in a
 *                 standalone container on the default docker network. The
 *                 simulated topology has no route to npmjs.org, so this
 *                 path deliberately runs outside it. This is what a
 *                 monorepo-free npm consumer actually gets (ce-fo6).
 *
 * Run as the last step of `cele2e build-infra`. After this:
 *   - celilo-e2e/management:vanilla — bun + unzip + bunfig (no celilo)
 *   - celilo-e2e/management:latest  — vanilla + celilo
 *
 * If the install regresses, this step fails build-infra loudly. The CLI is
 * verified AFTER the source-mount shim is written and with no /celilo mount,
 * so a broken shim fallback can't ship green (ISS-0124).
 */

import { execSync } from 'node:child_process';
import { join } from 'node:path';
import {
  RESTAGED_SIMS,
  ensureSharedInfra,
  recreateOnRebuiltImage,
  stopSharedInfra,
} from '../src/shared-infra';
import { network } from '../src/network-builder';
import { imageTag } from '../src/docker-compose-generator';
import { findMonorepoRoot } from '../src/repo-root';
import { detectStaleAcmeRoot, explainStaleRoot } from '../src/stale-acme-root';
import {
  packNpmRegistryTarballs,
  stageWebsiteDist,
} from '../src/stage-simulator-inputs';
import {
  INSTALLED_CLI_DIR,
  STAGED_TARBALL,
  digestScript,
  mismatchMessage,
  parseDigests,
} from '../src/installed-cli-digest';
import {
  CONSUMER_FINGERPRINT_PREFIX,
  PUBLISHED_FINGERPRINT_PREFIX,
  SOURCE_LABEL,
} from '../src/source-fingerprint';
import { readFileSync } from 'node:fs';
import { ZONE_GATEWAYS, greenwaveRouterIp } from '../src/types';

const PACKAGE_ROOT = join(import.meta.dir, '..');
const COMPOSE_FILE = join(PACKAGE_ROOT, 'docker-compose.test.yml');

function run(cmd: string): string {
  return execSync(cmd, { stdio: 'pipe', encoding: 'utf-8' });
}

interface ExecResult {
  exitCode: number;
  stdout: string;
  stderr: string;
}
type Exec = (cmd: string, timeoutMs?: number) => Promise<ExecResult>;

/**
 * Write the CLI shim to /usr/local/bin/celilo and verify the CLI works through
 * it. Shared by both bake modes.
 *
 * The shim (at /usr/local/bin/celilo so plain `docker exec bash -c celilo`
 * finds it) runs the INSTALLED CLI at /root/.bun/bin/celilo — the artifact
 * install.sh just produced, which is the thing the rig exists to test. It runs
 * the mounted /celilo workspace instead only when `CELILO_E2E_SOURCE_CLI` is
 * set, which `cele2e run --source-cli` does.
 *
 * That default used to be the other way round, and both halves of the swap are
 * worth stating.
 *
 * It cost real time per invocation. `bun run <source>` resolves 1522 modules
 * across the bind mount rather than starting an installed binary. Measured
 * inside this image on 2026-09-05, same command and same container, on a VM
 * sized per the host policy: `celilo --version` took 0.14s installed and 0.31s
 * from the mounted source over virtiofs, or 0.16s against 0.56s over sshfs. A
 * test issues 25 to 40 celilo commands.
 *
 * (An earlier measurement put that gap at 2.3s. Most of that was a separate
 * problem — the VM had been given 24 of the host's 32 GiB and the host was
 * swapping 21.9 GiB. Fixing the memory removed about 1.9s of it. Worth stating,
 * because a number measured with two variables moving blamed the wrong one.)
 *
 * And it made the verify below dishonest. This function exists because of
 * ISS-0124 — a broken shim fallback shipping green — and its comment says the
 * verify runs "with NO /celilo mount, i.e. exactly the shipped path". That was
 * true of the published bake, which runs in a standalone container, and false
 * of this one: the sim bake's container HAS the workspace mounted, so the old
 * shim sent the verify down the source branch and the installed CLI was never
 * executed at all. The check could not reach its subject. Now both modes verify
 * the installed binary.
 *
 * Two things must hold, both bitten in ISS-0124:
 *
 *  1. `bun add -g` links the celilo binary at /usr/local/bin/celilo as a
 *     SYMLINK into the package (bun's global bin dir in this image is
 *     /usr/local/bin, where bun itself lives). Writing the shim with
 *     `> /usr/local/bin/celilo` would follow that symlink and clobber the
 *     package's OWN launcher — turning the fallback into an infinite loop.
 *     So we `rm -f` the symlink first and write a fresh regular file.
 *  2. The fallback target /root/.bun/bin/celilo doesn't exist after a
 *     /usr/local/bin install, so we point a guaranteed symlink at the
 *     package launcher first. Guarded so an image where bun genuinely
 *     lives under /root/.bun is left untouched.
 *
 * Verify runs AFTER the shim and with NO /celilo mount — i.e. exactly the
 * shipped path — so a dangling/looping fallback fails the bake instead of
 * shipping green.
 */
async function installShimAndVerify(exec: Exec, label: string): Promise<string> {
  process.stdout.write('  source-mount shim ...... ');
  const t0 = Date.now();
  const shim = [
    '#!/bin/bash',
    '# Opt-in source loop: `cele2e run --source-cli` sets this in the container',
    '# environment. Unset (the default) runs the CLI install.sh actually installed.',
    'if [ -n "$CELILO_E2E_SOURCE_CLI" ] && [ -d /celilo/apps/celilo ]; then',
    '  exec bun run /celilo/apps/celilo/src/cli/index.ts "$@"',
    'fi',
    'exec /root/.bun/bin/celilo "$@"',
  ].join('\n');
  const shimBase64 = Buffer.from(shim).toString('base64');
  const shimResult = await exec(
    'mkdir -p /root/.bun/bin && ' +
      '{ [ -e /root/.bun/bin/celilo ] || ' +
      'ln -s /root/.bun/install/global/node_modules/@celilo/cli/bin/celilo /root/.bun/bin/celilo; } && ' +
      'rm -f /usr/local/bin/celilo && ' +
      `echo ${shimBase64} | base64 -d > /usr/local/bin/celilo && chmod +x /usr/local/bin/celilo`,
  );
  if (shimResult.exitCode !== 0) {
    throw new Error(`Shim install failed: ${shimResult.stderr}`);
  }
  console.log(`✔ ${Math.round((Date.now() - t0) / 1000)}s`);

  process.stdout.write('  verify celilo --version  ');
  const t1 = Date.now();
  const verify = await exec('celilo --version 2>&1');
  if (verify.exitCode !== 0) {
    console.log(`✗ exit ${verify.exitCode}`);
    console.error('--- celilo --version output ---');
    console.error(verify.stdout);
    console.error(verify.stderr);
    throw new Error(
      `celilo --version failed after shim (${label}) — image is broken (dangling shim fallback?)`,
    );
  }
  console.log(`✔ ${Math.round((Date.now() - t1) / 1000)}s  (${verify.stdout.trim()})`);
  return verify.stdout.trim();
}

/**
 * The image id currently on a tag, or '' when the tag is unused.
 *
 * Captured before a commit so the superseded image can be removed after it.
 * Every bake used to orphan a 1.5 GB untagged image and nothing ever collected
 * them: measured 2026-09-05, 3178 dangling images holding 25.8 GB, 1478 of them
 * from a single 35-hour window. That pile is what drives an operator to reach
 * for `docker image prune -a`, which deletes the TAGGED base images the next
 * build needs and costs a full 27-image rebuild.
 */
function imageIdOnTag(tag: string): string {
  try {
    return run(`docker image inspect --format '{{.Id}}' ${tag}`).trim();
  } catch {
    return '';
  }
}

/** Remove an image the bake just superseded. Best-effort: a tag or container still using it wins. */
function removeSupersededImage(previousId: string, tag: string): void {
  if (!previousId || previousId === imageIdOnTag(tag)) return;
  try {
    run(`docker image rm ${previousId}`);
    console.log(`  superseded image ....... ✔ removed ${previousId.slice(7, 19)}`);
  } catch {
    // Still referenced (a running container, another tag). Leaving it is correct.
  }
}

/**
 * The sim tarballs the default bake installs, staged fresh by
 * restageSimulatorInputs, carry a manifest recording the tree fingerprint they
 * were packed from. The stamp IS that value: the label then describes the
 * tarballs install.sh actually installs, not the tree the bake happened to run
 * in. Stamping `computeSourceFingerprint(tree)` here is what produced a green
 * doctor over a CLI without the tree's changes (celilo#1299): the tree moved
 * between the last `cele2e build` and the bake, and the fingerprint followed
 * the tree, not the stale cache the bake reinstalled.
 */
function simStampFromManifest(): string {
  const manifestPath = join(PACKAGE_ROOT, '.npm-registry-cache', 'pack-manifest.json');
  let manifest: { sourceFingerprint?: string | null };
  try {
    manifest = JSON.parse(readFileSync(manifestPath, 'utf-8')) as {
      sourceFingerprint?: string | null;
    };
  } catch {
    throw new Error(
      `No readable pack-manifest.json at ${manifestPath} — the sim tarballs were never staged. ` +
        'Run the bake from a monorepo checkout; restageSimulatorInputs writes the manifest.',
    );
  }
  if (!manifest.sourceFingerprint) {
    throw new Error(
      'pack-manifest.json carries no sourceFingerprint — the tarballs this bake installs cannot be attributed to a tree.',
    );
  }
  return manifest.sourceFingerprint;
}

/**
 * Stamp the source this image was baked from, so `cele2e doctor` can say
 * whether the CLI under test is the code in the working tree. See
 * `src/source-fingerprint.ts` for why that has to be measured, not remembered.
 */
/**
 * `celilo --version` prints "celilo <semver>", and `docker commit --change
 * "LABEL name=value"` splits on whitespace, so stamping the raw output fails
 * the commit with `Syntax error - can't find = in "2.2.0"`. Take the last
 * field, so the label carries the version alone (celilo#1318).
 */
function labelSafeVersion(version: string): string {
  return version.trim().split(/\s+/).pop() || 'unknown';
}

/**
 * The staged tarball for a package, as pack-celilo-packages recorded it.
 */
function stagedTarballPath(packageName: string): string {
  const manifestPath = join(PACKAGE_ROOT, '.npm-registry-cache', 'pack-manifest.json');
  const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8')) as {
    packages?: Array<{ name: string; tarball: string }>;
  };
  const entry = manifest.packages?.find((p) => p.name === packageName);
  if (!entry) {
    throw new Error(`pack-manifest.json lists no ${packageName} tarball to verify the install against.`);
  }
  // pack() records the basename; the tarballs sit beside the manifest.
  return join(PACKAGE_ROOT, '.npm-registry-cache', entry.tarball);
}

/**
 * Refuse to stamp a freshness label over a CLI that is not the one we packed.
 * See src/installed-cli-digest.ts for why the label alone cannot say this.
 */
async function verifyInstalledCliIsStaged(exec: Exec, containerId: string): Promise<void> {
  process.stdout.write('  installed CLI = staged   ');
  const t = Date.now();
  run(`docker cp ${JSON.stringify(stagedTarballPath('@celilo/cli'))} ${containerId}:${STAGED_TARBALL}`);
  const b64 = Buffer.from(digestScript(INSTALLED_CLI_DIR)).toString('base64');
  const probe = await exec(
    `echo ${b64} | base64 -d > /tmp/verify-cli.sh && bash /tmp/verify-cli.sh 2>&1`,
    120_000,
  );
  const digests = parseDigests(`${probe.stdout}${probe.stderr}`);
  if (digests.staged !== digests.installed) {
    console.log('\u2717');
    throw new Error(mismatchMessage(digests));
  }
  console.log(`\u2714 ${Math.round((Date.now() - t) / 1000)}s  (${digests.staged.slice(0, 16)})`);
}

function sourceStamp(published: boolean, version: string): string {
  if (published) return `${PUBLISHED_FINGERPRINT_PREFIX}${labelSafeVersion(version)}`;
  // Consumer mode: restageSimulatorInputs is the only writer of
  // pack-manifest.json and it is skipped without a checkout, so there is no
  // tree fingerprint to read — and none to record, since the tree does not
  // exist. Stamp the version the bake just installed and verified (celilo#1318).
  if (!findMonorepoRoot(PACKAGE_ROOT)) {
    return `${CONSUMER_FINGERPRINT_PREFIX}${labelSafeVersion(version)}`;
  }
  return simStampFromManifest();
}

/**
 * Re-stage the simulator inputs the default bake's install path depends on,
 * and rebuild the two sim images that COPY them at docker-build time.
 *
 * Without this, the bake reinstalls whatever tarballs the last `cele2e build`
 * left in `.npm-registry-cache/` — measured 2026-09-07: a bake at 00:17 UTC
 * shipped @celilo/cli 2.1.0 from a cache packed at 15:18 the day before,
 * silently dropping a CLI fix that had landed at 19:48 (celilo#1299). The
 * rebuilds are not optional either: the per-test stack only builds MISSING
 * images, so fresh tarballs on disk never reach the running sim otherwise.
 * Both docker builds are cache-warm no-ops when nothing staged has changed.
 */
function restageSimulatorInputs(): void {
  const repoRoot = findMonorepoRoot(PACKAGE_ROOT);
  if (!repoRoot) {
    // Consumer mode (no checkout): there is no tree to repack, and nothing to
    // correct. stageFromPublic already fetched THIS run's tarballs and site
    // dist into the sim caches, and the sim images were built from them before
    // the bake started. celilo#1299 is a monorepo-mode hazard — there the bake
    // would otherwise reinstall the previous build's cached CLI — so the
    // restage is redundant here, not impossible-but-required. Throwing broke
    // npm-consumer-smoke, whose whole point is a bake with no monorepo source
    // (celilo#1318).
    console.log('  restage ................ skipped (consumer mode: inputs staged this run)');
    return;
  }
  stageWebsiteDist(repoRoot, PACKAGE_ROOT);
  packNpmRegistryTarballs(repoRoot, PACKAGE_ROOT);
  for (const { dockerfile } of RESTAGED_SIMS) {
    const tag = imageTag(dockerfile);
    process.stdout.write(`  rebuild ${tag.padEnd(22)} `);
    const t0 = Date.now();
    execSync(
      `docker build -q -f ${join(PACKAGE_ROOT, dockerfile)} -t ${tag} ${PACKAGE_ROOT}`,
      { stdio: 'pipe' },
    );
    console.log(`✔ ${Math.round((Date.now() - t0) / 1000)}s`);
  }
}

/**
 * Default bake: install.sh against the simulated celilo.computer / npm
 * registry, inside the standard topology.
 */
/**
 * The probe waits for HTTPS to VERIFY, so a trust failure and an unfinished
 * issuance both spend the whole budget and both used to print "timed out".
 * They need opposite responses, so branch the headline on curl's own exit code
 * rather than making the reader infer it from the layers (crew-builder,
 * celilo#1427).
 */
/**
 * The TLS/DNS layers, read from inside the management container.
 *
 * Hoisted out of the probe's failure path because the probe is not the only
 * step that can fail on TLS: install.sh's own curl-pipe-bash talks to the same
 * host a moment later, and when THAT failed the bake printed the curl error
 * alone with no layers under it (measured, run 7255: `curl: (35) TLS connect
 * error ... tlsv1 alert internal error`, one second after the probe had
 * verified the same URL successfully). A failure with no layers is the exact
 * shape celilo#1365 spent two weeks in.
 */
async function tlsDiagnosticLayers(exec: Exec, projectName?: string): Promise<string> {
  const diag = async (label: string, cmd: string): Promise<string> => {
    const out = await exec(cmd, 10_000);
    return `--- ${label} ---\n${out.stdout.trim() || '(no output)'}`;
  };
  return [
    await diag('resolv.conf', 'cat /etc/resolv.conf'),
    await diag('dig @10.226.1.10 (internal resolver)', 'dig +time=2 +tries=1 @10.226.1.10 celilo.computer'),
    await diag('dig @203.0.113.1 (public fallback)', 'dig +time=2 +tries=1 @203.0.113.1 celilo.computer'),
    await diag('trusted CA files', 'ls -la /usr/local/share/ca-certificates/'),
    await diag('Pebble runtime root reachable from management', 'curl -sk --max-time 5 https://100.64.0.100:15000/roots/0 | head -c 300; echo'),
    await diag('fetched runtime root (if any)', 'openssl x509 -in /usr/local/share/ca-certificates/pebble-acme-root.crt -noout -subject -enddate 2>/dev/null'),
    await diag('cert chain served by website-sim', 'timeout 8 openssl s_client -connect celilo.computer:443 -servername celilo.computer -showcerts </dev/null 2>/dev/null | grep -E "^ [0-9] s:|^   i:"'),
    await diag('curl -v (DNS/connect/TLS/HTTP stages)', 'curl -v --max-time 5 https://celilo.computer/install.sh -o /dev/null 2>&1 | tail -20'),
    // The probe's EXACT command, re-run here. Runs 7255-7260 all showed the
    // probe returning 0 and install.sh's curl failing 130ms later on the same
    // host, which is either two different answers from one server or a probe
    // reading something other than what it thinks. Printing both side by side
    // decides it, instead of reasoning about which flag matters.
    await diag(
      're-run of the probe command',
      "curl -sS --max-time 3 -o /dev/null -w 'curl-exit=%{exitcode}\n' https://celilo.computer/install.sh 2>&1",
    ),
    await diag(
      'the same URL with -f, as install.sh fetches it',
      "curl -fsS --max-time 5 -o /dev/null -w 'curl-exit=%{exitcode}\n' https://celilo.computer/install.sh 2>&1",
    ),
    // Did management's own startup script reach its Pebble-root fetch at all?
    // management-routes.sh runs under `set -e` with six `ip route add` calls
    // AHEAD of that fetch (line 61), and a route add exiting 2 has killed this
    // script before — celilo#1351, recorded in docker-compose-generator.ts. If
    // the trust store has no runtime root, this log says whether the fetch
    // failed, is still looping, or was never reached.
    await diag('is /startup.sh still running?', 'ps -ef | grep -c "[s]tartup.sh"'),
    managementStartupLog(projectName),
    // Every layer above is read from INSIDE management, so all of them describe
    // the client. When the server aborts the handshake before sending a
    // certificate (run 7258), the client has nothing left to report and the
    // reason is only in Caddy's own log — whether it got an ACME cert at all.
    websiteSimLog(),
  ].join('\n');
}

/** The per-test management container's own startup output, read from the host. */
function managementStartupLog(projectName?: string): string {
  if (!projectName) return '--- management startup log ---\n(project name unavailable)';
  try {
    const id = execSync(`docker ps -aq --filter name=${projectName} --filter name=management`, {
      encoding: 'utf-8',
      timeout: 15_000,
    })
      .trim()
      .split('\n')[0];
    if (!id) return '--- management startup log ---\n(container not found)';
    const out = execSync(`docker logs --tail 40 ${id} 2>&1`, {
      encoding: 'utf-8',
      timeout: 20_000,
    });
    return `--- management startup log (server side) ---\n${out.trim() || '(no output)'}`;
  } catch (err) {
    return `--- management startup log ---\n(unreadable: ${(err as Error).message})`;
  }
}

/** celilo-website-sim's own Caddy log, read from the host. */
function websiteSimLog(): string {
  try {
    const out = execSync(
      'docker compose -f docker-compose.shared.yml -p celilo-e2e-shared logs --tail 40 celilo-website 2>&1',
      // The package's OWN root, not the monorepo's: npm-consumer-smoke runs this
      // from node_modules outside any checkout, where findMonorepoRoot() is
      // undefined and join() throws (measured, run 7260). Same resolution
      // shared-infra.ts:getE2eDir uses, for the same reason.
      { cwd: join(import.meta.dir, '..'), encoding: 'utf-8', timeout: 20_000 },
    );
    return `--- celilo-website-sim log (server side) ---\n${out.trim() || '(no output)'}`;
  } catch (err) {
    return `--- celilo-website-sim log (server side) ---\n(unreadable: ${(err as Error).message})`;
  }
}

/** Sentinel: the probe ran but its `curl-exit=` marker was not in the output. */
const CA_WAIT_SECONDS = 240;
const PROBE_UNREADABLE = -2;

function explainProbeFailure(curlExit: number): string {
  // 60 unknown issuer, 35 TLS connect, 51 peer cert not ok.
  if ([60, 35, 51].includes(curlExit)) {
    return (
      `celilo.computer served a certificate this container cannot verify (curl ${curlExit}). ` +
      "The step above already waited for management's runtime root, so this is the STALE-ROOT " +
      'case, not the not-yet-fetched one — those need opposite responses and exit 60 alone ' +
      'cannot tell them apart (celilo#1365 spent runs on the wrong one). ' +
      'This is NOT a timeout and waiting longer cannot fix it. Pebble mints a random root CA ' +
      'at every startup and Caddy caches its cert until expiry, so a Pebble that restarted ' +
      'without the website sim restarting is serving a cert from a root that no longer exists ' +
      '(celilo#1427). Check the chain above against the live root. Remedy: `cele2e down` then ' +
      '`cele2e build-infra`, so Pebble and Caddy start together.'
    );
  }
  // 6 could not resolve, 7 could not connect, 28 operation timeout.
  if ([6, 7, 28].includes(curlExit)) {
    return (
      `celilo.computer could not be reached at all (curl ${curlExit}) within 30s — a resolution ` +
      'or connection failure, not a trust one. Either the website sim is down or its ACME ' +
      'issuance genuinely has not finished. Read the DNS layers above.'
    );
  }
  if (curlExit === PROBE_UNREADABLE) {
    return (
      'The HTTPS probe never returned a readable curl exit code in 30s — its ' +
      '`curl-exit=` marker was missing from both streams. Treated as NOT ready: ' +
      'the probe cannot prove the site is serving, and a probe that cannot fail ' +
      'is how celilo#1365 stayed invisible. See the layers above.'
    );
  }
  return (
    `Timed out (${Math.round(CA_WAIT_SECONDS)}s of wall clock) waiting for celilo.computer to serve install.sh over HTTPS (curl ${curlExit}). ` +
    'See the DNS / curl -v layers above for which one failed. A cold shared-infra ' +
    'caddy needs ~90s to obtain its FIRST certificate, so this is a real failure ' +
    'rather than an impatient probe.'
  );
}

async function bakeViaSim(): Promise<void> {
  console.log('▸ Baking celilo into management:latest via install.sh');
  console.log('');

  // Fresh tarballs and site dist, and sim images that actually carry them,
  // BEFORE install.sh runs — otherwise this bake reinstalls the last build's
  // CLI (celilo#1299). And before shared infra starts, so a stack started here
  // is created from the images this bake built (celilo#1502).
  restageSimulatorInputs();

  process.stdout.write('  shared-infra ........... ');
  const t0 = Date.now();
  await ensureSharedInfra();
  console.log(`✔ ${Math.round((Date.now() - t0) / 1000)}s`);

  // A stack that was ALREADY up is the other half of celilo#1502: the last
  // successful bake left it running, on the sim images that bake built.
  recreateOnRebuiltImage();

  process.stdout.write('  per-test stack ......... ');
  const t1 = Date.now();
  const net = await network().topology('default').managementVariant('vanilla').start();
  console.log(`✔ ${Math.round((Date.now() - t1) / 1000)}s  (${net.projectName})`);

  const exec: Exec = (cmd, timeoutMs) => net.exec('management', cmd, timeoutMs);

  let containerId = '';
  let bakeFailed = false;
  try {
    // Wait for the management container's Pebble CA fetch to complete so
    // the curl-pipe-bash below can verify TLS to celilo.computer. Without
    // pipefail an early curl failure returns 0 silently and install.sh
    // "runs" in 0s installing nothing.
    // WAIT FOR THE DEPENDENCY THIS STEP IS NAMED AFTER, not for a symptom of it.
    // management-routes.sh fetches Pebble's runtime root in a FOREGROUND retry
    // loop of 60 tries x 2s (config/routing/management-routes.sh:61-71) — worst
    // case 120s, and it only starts once the per-test routing is up. The HTTPS
    // probe below budgeted 30s, so it gave up while that fetch was still
    // running and blamed the certificate. Measured on run 7266: /startup.sh
    // still running, its log stopped at "Waiting for Pebble ACME root CA..."
    // with no "installed" line, and the trust store held only build-time certs
    // — while the website sim had already obtained a valid cert. The two waits
    // were simply sized wrong relative to each other.
    process.stdout.write('  pebble root installed .... ');
    const tRoot = Date.now();
    let rootReady = false;
    // 240s, not the 150 this was first written with. MEASURED on run 7268:
    // the fetch took 136s — past the loop's nominal 120s ceiling, because each
    // failing curl spends its own connect timeout before the 2s sleep. 150
    // left 10% headroom on an idle builder, which is how a green gate starts
    // flaking under load. The budget costs nothing when the root arrives fast:
    // the wait ends the moment the file appears.
    for (let i = 0; i < 240; i++) {
      const probe = await exec(
        'test -s /usr/local/share/ca-certificates/pebble-acme-root.crt && echo root-present || echo root-absent',
        5_000,
      );
      if (`${probe.stdout}${probe.stderr}`.includes('root-present')) {
        rootReady = true;
        break;
      }
      await new Promise((resolve) => setTimeout(resolve, 1000));
    }
    if (!rootReady) {
      console.log('✗');
      console.error(await tlsDiagnosticLayers(exec, net.projectName));
      throw new Error(
        "management never installed Pebble's runtime ACME root in 240s. Its startup script " +
          'fetches it in a foreground loop before celilo.computer can be verified at all; the ' +
          'layers above say whether that loop is still running, died early, or is failing its ' +
          'fetch (config/routing/management-routes.sh:61).',
      );
    }
    console.log(`✔ ${Math.round((Date.now() - tRoot) / 1000)}s`);

    process.stdout.write('  pebble CA ready .......... ');
    const tCa = Date.now();
    let caReady = false;
    let lastProbeExit = -1;
    let lastProbeOutput = '';
    // Re-checked while INCONCLUSIVE, at most every 30s. A single check at t=0
    // sits at the flakiest moment in the run (shared infra has just come up), and
    // an exec that answers nothing there scores `unknown` — so a check that fired
    // only once would silently fall back to burning the whole budget, which is the
    // exact defect this is here to remove.
    let staleCheckAt = 0;
    // WALL CLOCK, not an iteration count. Each pass costs up to 3s of curl plus a
    // 1s sleep, so `i < 30` was never 30 seconds — and the error text said "30s"
    // regardless, which is how this budget got read as generous when it was not.
    // Measured 2026-09-26 on a torn-down host: shared-infra caddy needs ~90s to
    // obtain its FIRST certificate (caddy's own log: "certificate obtained
    // successfully" at 90.9s elapsed, after one tls-alpn-01 retry). A 30-iteration
    // cap could not reach that, so a cold rig failed the bake three runs running
    // while the website was merely still issuing. 240s covers the measurement with
    // margin and still fails in well under the suite's own budget.
    const CA_WAIT_MS = 240_000;
    const caDeadline = Date.now() + CA_WAIT_MS;
    while (Date.now() < caDeadline) {
      // NO `$?` HERE, EVER. dockerExec wraps the command as
      // `bash -c ${JSON.stringify(cmd)}` (container-manager.ts:636) and hands
      // that to a HOST shell, which expands `$?` itself — to its own last
      // status, which is 0. So `echo "curl-exit=$?"` reported curl-exit=0 on
      // every run this bead has ever had, including the ones where curl printed
      // `(60) SSL certificate problem: unable to get local issuer certificate`
      // two lines above it (run 7262 prints exactly that pair). The probe was
      // structurally incapable of failing, which is why celilo#1365 read as a
      // 30s timeout and then as a green step: the step never had an opinion.
      // curl's own %{exitcode} needs no shell variable and cannot be expanded
      // away.
      const probe = await exec(
        "curl -sS --max-time 3 -o /dev/null -w 'curl-exit=%{exitcode}\n' https://celilo.computer/install.sh 2>&1",
        5_000,
      );
      const text = `${probe.stdout}\n${probe.stderr}`.trim();
      const code = /curl-exit=(\d+)/.exec(text);
      // FAIL CLOSED. The probe command ends in `echo`, so the shell's exit is 0
      // whatever curl did — falling back to probe.exitCode scored "I could not
      // read the answer" as "ready". Measured on run 7258: the probe reported
      // `✔ 0s` while celilo.computer was serving NO certificate at all
      // (openssl s_client returned an empty chain and the server aborted the
      // handshake with `internal error (592)`), and the real failure surfaced
      // one step later as install.sh's curl (35). A probe that cannot fail is
      // worse than no probe: it converted celilo#1365's defect into a green
      // step plus a mystery. Unreadable now counts as not-ready, the same bias
      // live-stack.ts already applies to its own evidence test.
      lastProbeExit = code ? Number(code[1]) : PROBE_UNREADABLE;
      lastProbeOutput = text;
      if (lastProbeExit === 0) {
        caReady = true;
        break;
      }
      // REFUSE INSTANTLY on a dead root rather than spending the 240s budget
      // and then reporting a timeout (celilo#1427). The chain either names
      // Pebble's live root or it does not, and that answer does not change by
      // waiting. A cold caddy serving NOTHING yet scores `unknown` and keeps
      // the wait, which is the case the budget exists for.
      if (Date.now() >= staleCheckAt) {
        staleCheckAt = Date.now() + 30_000;
        const [live, served] = await Promise.all([
          exec('curl -sk --max-time 5 https://100.64.0.100:15000/roots/0 | openssl x509 -noout -subject 2>/dev/null', 10_000),
          // Deliberately a LOOSE grep: openssl's indentation for the issuer line
          // differs across versions, and detectStaleAcmeRoot does the real
          // parsing anyway. An anchored pattern that misses scores `unknown`,
          // which reads exactly like a cold caddy and disables the check.
          exec('timeout 8 openssl s_client -connect celilo.computer:443 -servername celilo.computer -showcerts </dev/null 2>/dev/null | grep i:', 12_000),
        ]);
        const verdict = detectStaleAcmeRoot(live.stdout, served.stdout);
        if (verdict.kind === 'stale') {
          console.log('✗');
          console.error(await tlsDiagnosticLayers(exec, net.projectName));
          throw new Error(explainStaleRoot(verdict));
        }
        // Settled: the served chain names the live root, so it cannot go stale
        // by waiting. Stop paying ~20s of exec per cycle for a known answer.
        if (verdict.kind === 'current') staleCheckAt = Number.POSITIVE_INFINITY;
      }
      await new Promise((resolve) => setTimeout(resolve, 1000));
    }
    if (!caReady) {
      // Self-diagnosing (e2e-confidence #255, mirroring install-sh.test.ts):
      // a bake that fails here used to name two GUESSES and discard the probe
      // output, so celilo#1365 produced two CI failures with no cause visible.
      // Dump the DNS / reachability / TLS layers from inside the container.
      const layers = await tlsDiagnosticLayers(exec, net.projectName);
      console.error(layers);
      console.error(`--- last probe (curl exit ${lastProbeExit}) ---\n${lastProbeOutput || '(no output)'}`);
      throw new Error(explainProbeFailure(lastProbeExit));
    }
    console.log(`✔ ${Math.round((Date.now() - tCa) / 1000)}s`);

    process.stdout.write('  install.sh ............. ');
    const t2 = Date.now();
    const result = await exec(
      'set -o pipefail; curl -fsSL https://celilo.computer/install.sh | bash 2>&1',
      180_000,
    );
    if (result.exitCode !== 0) {
      console.log(`✗ exit ${result.exitCode}`);
      console.error('--- install.sh output (tail) ---');
      console.error(result.stdout.slice(-3000));
      console.error(result.stderr.slice(-2000));
      // The probe verified this very URL seconds ago, so a TLS failure HERE is
      // a change between the two calls, not a static misconfiguration — and it
      // is unreadable without the layers (run 7255).
      console.error(await tlsDiagnosticLayers(exec, net.projectName));
      throw new Error('install.sh failed inside the bake container');
    }
    console.log(`✔ ${Math.round((Date.now() - t2) / 1000)}s`);

    // Shim + verify. Post-shim and, since the shim now prefers the installed
    // CLI, genuinely against the installed binary rather than the mount.
    const version = await installShimAndVerify(exec, 'sim');

    containerId = run(
      `docker compose -f ${COMPOSE_FILE} -p ${net.projectName} ps -q management`,
    ).trim();
    if (!containerId) {
      throw new Error('Could not resolve management container id');
    }
    // CONSUMER MODE HAS NOTHING TO VERIFY AGAINST, and the check reads a file
    // that is deliberately not shipped. verifyInstalledCliIsStaged compares the
    // installed CLI against the tarball restageSimulatorInputs packed, recorded
    // in PACKAGE_ROOT/.npm-registry-cache/pack-manifest.json. Without a checkout
    // that restage is skipped (see restageSimulatorInputs), nothing writes the
    // manifest, and `.npm-registry-cache` is excluded from the e2e tarball on
    // purpose (celilo#116) — so PACKAGE_ROOT resolves into node_modules and the
    // read is a guaranteed ENOENT. sourceStamp already reasons exactly this way
    // a few lines up.
    //
    // It is not merely impossible here, it is meaningless: in consumer mode the
    // CLI came from the registry this run, so there is no staged copy for it to
    // have drifted from. The freshness label records CONSUMER_FINGERPRINT for
    // the same reason.
    //
    // Measured 2026-09-26: npm-consumer-smoke run 7335 died at 174s with
    // `ENOENT ... node_modules/@celilo/e2e/.npm-registry-cache/pack-manifest.json`,
    // which failed PR #1437 and blocked the release the fleet was waiting on.
    // The check landed in 00e2367d for celilo#1423 (image-freshness certifying a
    // stale CLI) — a real fix whose guard was written for monorepo mode only.
    if (findMonorepoRoot(PACKAGE_ROOT)) {
      await verifyInstalledCliIsStaged(exec, containerId);
    } else {
      console.log('  installed CLI = staged   skipped (consumer mode: nothing was staged to compare)');
    }

    process.stdout.write('  docker commit .......... ');
    const t5 = Date.now();
    const superseded = imageIdOnTag('celilo-e2e/management:latest');
    // The committed container runs under the sim topology's compose env, and
    // commit captures its Env. Since the default managementZone became
    // secure-mgmt, that baked FW_MAIN_HOP=10.226.120.1 (and the control-plane
    // default gateway) into :latest — every internal-topology suite then
    // inherited a nexthop that is not on-link from the internal LAN and the
    // mgmt box crash-looped in management-routes.sh (celilo#1351). Reset the
    // per-topology routing vars to the internal-topology defaults at commit
    // time, so the image is topology-neutral; every compose sets both anyway.
    run(
      [
        'docker commit',
        `--change ${JSON.stringify(`ENV DEFAULT_GATEWAY=${greenwaveRouterIp()}`)}`,
        `--change ${JSON.stringify(`ENV FW_MAIN_HOP=${ZONE_GATEWAYS.internal}`)}`,
        `--change ${JSON.stringify(`LABEL ${SOURCE_LABEL}=${sourceStamp(false, version)}`)}`,
        containerId,
        'celilo-e2e/management:latest',
      ].join(' '),
    );
    console.log(`✔ ${Math.round((Date.now() - t5) / 1000)}s`);
    removeSupersededImage(superseded, 'celilo-e2e/management:latest');
  } catch (err) {
    bakeFailed = true;
    throw err;
  } finally {
    process.stdout.write('  tearing down ........... ');
    const tDown = Date.now();
    await net.stop();
    console.log(`✔ ${Math.round((Date.now() - tDown) / 1000)}s`);
    if (bakeFailed) {
      // A failed bake must not leave the shared stack live. On the builder the
      // job dies at this step and the workflow's `if: always()` teardown steps
      // never run (observed on runs 6645/6682/6693/6698: the log ends at
      // "Cleaning up container for job"), so the 13-container shared stack
      // then refuses every full run for the next 60 minutes under the
      // orphan-age gate in live-stack.ts. Success keeps the stack: the Mac dev
      // flow reuses it and the workflow's teardown step owns the builder's
      // success path. (celilo#1365, celilo#1320)
      try {
        await stopSharedInfra();
      } catch {
        // best-effort: the workflow teardown or the next run's sweep is the
        // backstop, and hiding the reason here beats failing twice.
      }
    }
  }

  console.log('');
  console.log('✓ celilo-e2e/management:latest baked from install.sh');
}

/**
 * Published bake: install the PUBLISHED @celilo/cli from REAL npm in a
 * standalone vanilla container on the default docker network (real
 * internet). The simulated topology has no route to npmjs.org, so we run
 * outside it.
 */
async function bakeViaPublished(): Promise<void> {
  console.log('▸ Baking PUBLISHED @celilo/cli into management:latest from real npm');
  console.log('');

  const name = 'celilo-bake-published';
  const dexec: Exec = (cmd) => {
    try {
      const stdout = execSync(`docker exec ${name} bash -c ${JSON.stringify(cmd)}`, {
        stdio: 'pipe',
        encoding: 'utf-8',
      });
      return Promise.resolve({ stdout, stderr: '', exitCode: 0 });
    } catch (e: unknown) {
      const err = e as { stdout?: Buffer; stderr?: Buffer; status?: number };
      return Promise.resolve({
        stdout: err.stdout?.toString() ?? '',
        stderr: err.stderr?.toString() ?? '',
        exitCode: err.status ?? 1,
      });
    }
  };

  // Fresh container every bake — a stale one may have half-applied global state.
  try {
    run(`docker rm -f ${name}`);
  } catch {
    // no stale container — fine
  }

  process.stdout.write('  start vanilla container  ');
  const t0 = Date.now();
  // Default bridge network → real internet (real npm, real DNS).
  run(`docker run -d --name ${name} celilo-e2e/management:vanilla sleep infinity`);
  console.log(`✔ ${Math.round((Date.now() - t0) / 1000)}s`);

  try {
    // Reset bun global state before installing from real npm:
    //  - the @celilo scope override (bunfig) pins @celilo to the sim
    //    registry (npm-registry.lab), unresolvable on the default bridge —
    //    move it aside so `bun add -g` reaches real npm.
    //  - a prior monorepo bake leaves bun.lock pinning @celilo tarball URLs
    //    to the sim registry; every later `bun add -g` then dies with
    //    FailedToOpenSocket regardless of cache clearing (ce-fo6). Nuke it.
    process.stdout.write('  reset bun state ........ ');
    const t1 = Date.now();
    const reset = await dexec(
      'mv /root/.bunfig.toml /root/.bunfig.toml.sim 2>/dev/null; ' +
        'rm -f /root/.bun/install/global/bun.lock; true',
    );
    if (reset.exitCode !== 0) {
      throw new Error(`Failed to reset bun state: ${reset.stderr}`);
    }
    console.log(`✔ ${Math.round((Date.now() - t1) / 1000)}s`);

    process.stdout.write('  bun add -g (real npm) .. ');
    const t2 = Date.now();
    const add = await dexec(
      'bun add -g @celilo/cli@latest @celilo/event-bus@latest @celilo/e2e@latest 2>&1',
    );
    if (add.exitCode !== 0) {
      console.log(`✗ exit ${add.exitCode}`);
      console.error('--- bun add -g output ---');
      console.error(add.stdout.slice(-3000));
      console.error(add.stderr.slice(-2000));
      throw new Error('bun add -g of published @celilo/* from real npm failed');
    }
    console.log(`✔ ${Math.round((Date.now() - t2) / 1000)}s`);

    // Shim + verify (verify is post-shim, no mount — the shipped path).
    const version = await installShimAndVerify(dexec, 'published');

    // Restore the sim @celilo scope override so the committed image is
    // drop-in compatible with the simulated topology it runs tests in.
    // ponytail: inert at test runtime (nothing re-fetches @celilo), kept
    // only for parity with the sim-baked image.
    await dexec('mv /root/.bunfig.toml.sim /root/.bunfig.toml 2>/dev/null; true');

    process.stdout.write('  docker commit .......... ');
    const t3 = Date.now();
    // The vanilla container was started with `sleep infinity`; docker commit
    // persists the running Cmd, so override it back to the management image's
    // real entrypoint (Dockerfile.management: CMD ["/startup.sh"], no
    // ENTRYPOINT). Without this the baked image never runs /startup.sh ->
    // ssh-keys volume stays empty -> fw-main crashloops (ce-um6).
    const superseded = imageIdOnTag('celilo-e2e/management:latest');
    run(
      `docker commit --change 'CMD ["/startup.sh"]' --change ${JSON.stringify(`LABEL ${SOURCE_LABEL}=${sourceStamp(true, version)}`)} ${name} celilo-e2e/management:latest`,
    );
    console.log(`✔ ${Math.round((Date.now() - t3) / 1000)}s`);
    removeSupersededImage(superseded, 'celilo-e2e/management:latest');
  } finally {
    process.stdout.write('  removing container ..... ');
    const t4 = Date.now();
    try {
      run(`docker rm -f ${name}`);
    } catch {
      // best-effort
    }
    console.log(`✔ ${Math.round((Date.now() - t4) / 1000)}s`);
  }

  console.log('');
  console.log('✓ celilo-e2e/management:latest baked from PUBLISHED @celilo/cli');
}

async function main(): Promise<void> {
  const published = process.argv.includes('--published');
  if (published) {
    await bakeViaPublished();
  } else {
    await bakeViaSim();
  }
}

main().catch((err) => {
  console.error('');
  console.error(err);
  process.exit(1);
});