import {
  type ChildProcessWithoutNullStreams,
  type ExecSyncOptions,
  type SpawnOptionsWithoutStdio,
  execSync,
  spawn,
} from 'node:child_process';
import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { basename, join, resolve } from 'node:path';
import { parseIpv4, subnetContains } from '@celilo/capabilities';
import { parse as parseYaml } from 'yaml';
import { startBrowser } from './browser';
import { MIN_CLI_VERSION, checkCliVersion } from './cli-version-contract';
import {
  type FirewallLeg,
  SHARED_PROJECT_NAME,
  firewallZoneLegs,
  generateSharedInfraYaml,
  generateTestComposeYaml,
  getAllMachines,
  referencedImages,
  registryUploadsHostDir,
} from './docker-compose-generator';
import { type ModuleHost, parseModuleHost, parseModuleWhere } from './module-host';
import { CONTAINER_PREFIX, GUEST_PROJECT_LABEL } from './proxmox-provisioner';
import { ensureSharedInfra } from './shared-infra';
import { SIMULATOR_IPS } from './simulator-ips';
import { startSocksProxy } from './socks-proxy';
import type {
  BrowserHandle,
  BrowserOptions,
  ExecResult,
  NetworkConfig,
  NetworkHandle,
  ProxyOptions,
  SocksProxyHandle,
  TopologyPreset,
} from './types';
import {
  CeliloCommandError,
  ZONE_GATEWAYS,
  ZONE_SUBNETS,
  greenwaveRouterIp,
  internalNatIp,
} from './types';

/**
 * Every way this module reaches the docker CLI, injectable so the cleanup
 * sweeps, the exec wrappers and the image accounting are unit-testable
 * without a daemon (ce-h4no, lane A1 of openspec/changes/e2e-suite-recovery).
 *
 * Follows the DoctorProbe pattern in doctor.ts: an interface of typed
 * operations with the real implementation as the default, so no caller
 * changes. proxmox-provisioner.ts owns the narrower `DockerRunner` name
 * (argv-only, sync, no streaming), so this one is named for what it wraps:
 * the docker CLI itself. Tests substitute a fake via `withDockerCli`.
 */
export interface DockerCli {
  /** Synchronous `docker ...` shell command. Throws on non-zero exit, like execSync. */
  exec(command: string, opts?: ExecSyncOptions): string;
  /**
   * Streaming `docker ...` for long-running compose build and exec. Callers
   * never pass stdio (pipe default), so the streams are non-null.
   */
  spawn(args: string[], opts?: SpawnOptionsWithoutStdio): ChildProcessWithoutNullStreams;
}

export const realDockerCli: DockerCli = {
  exec(command, opts) {
    // Encoding is pinned last so TS narrows the return to `string` even if
    // a caller passes an opts object that could in principle override it.
    return execSync(command, { ...opts, encoding: 'utf-8' });
  },
  spawn(args, opts) {
    return spawn('docker', args, opts);
  },
};

let dockerCli: DockerCli = realDockerCli;

/**
 * Run `fn` with `runner` as this module's docker access, restoring the
 * previous runner afterwards — even when `fn` throws. A sync callback runs
 * and restores synchronously; a promise callback keeps the override in force
 * across its awaits and restores it when it settles. Concurrent async scopes
 * would restore out of order, so tests await one scope at a time.
 */
export function withDockerCli<T>(runner: DockerCli, fn: () => T): T;
export function withDockerCli<T>(runner: DockerCli, fn: () => Promise<T>): Promise<T>;
export function withDockerCli<T>(runner: DockerCli, fn: () => T | Promise<T>): T | Promise<T> {
  const previous = dockerCli;
  dockerCli = runner;
  try {
    const result = fn();
    if (result instanceof Promise) {
      return result.finally(() => {
        dockerCli = previous;
      });
    }
    dockerCli = previous;
    return result;
  } catch (err) {
    dockerCli = previous;
    throw err;
  }
}

/** Package root — where docker/, config/, simulators/ live */
const PACKAGE_ROOT = join(__dirname, '..');

/**
 * The `.netapp` build-infra already staged for this module, if nothing in the
 * source has changed since.
 *
 * `publishModule` used to repackage from source unconditionally, inside a
 * running stack, under a 60s cap. Measured 2026-09-04: build-infra packaged
 * `celilo-registry` (an 82MB netapp) in 11 seconds before any stack existed,
 * then publishModule exceeded 60 seconds on identical work because the stack was
 * up and eating the host. `registry-pipeline` failed on exactly that, and it is
 * not one of the three flakes that suite is quarantined for (celilo#1258).
 *
 * `find -newer -quit` asks the filesystem instead of walking in JS and stops at
 * the first newer file. A staleness check that cannot answer repackages: reusing
 * on an inconclusive result is how a test silently runs against a stale module.
 */
function stagedNetappIfCurrent(moduleDir: string, moduleId: string): string | null {
  const staged = join(PACKAGE_ROOT, 'netapps', `${moduleId}.netapp`);
  if (!existsSync(staged)) return null;

  // Only paths the package actually contains can make it stale. `e2e/` is
  // excluded from a module package wholesale and `node_modules/.bin` is excluded
  // from the hook runtime closure — see `classifyModulePath` and
  // `includeNodeModulesPath` in apps/celilo/src/module/packaging/, which are the
  // authority. They are restated rather than imported because packages/e2e has
  // no import path into apps/celilo, and registry-server's bootstrap.ts already
  // carries the same duplication for the same reason.
  //
  // Getting these wrong is safe in one direction only, and it is this one:
  // counting a non-packaged path makes us repackage needlessly (slow), while
  // MISSING a packaged path would reuse a stale netapp (wrong). Every exclusion
  // here is a path the packager does not ship, so it cannot cause the latter.
  // Without them nothing is ever reused: `bun install` touches
  // `e2e/node_modules/.bin/*` and every module looks permanently dirty.
  try {
    const newer = execSync(
      `find ${JSON.stringify(moduleDir)} -newer ${JSON.stringify(staged)} -not -path '*/e2e/*' -not -path '*/node_modules/.bin/*' -print -quit`,
      { encoding: 'utf-8', timeout: 30_000 },
    ).trim();
    return newer === '' ? staged : null;
  } catch {
    return null;
  }
}

/**
 * Find the Celilo project root by walking up from the package directory.
 * Looks for apps/celilo/ as the marker.
 */
/**
 * Try to find the Celilo project root.
 *
 * Resolution order:
 * 1. CELILO_E2E_INFRA_ROOT env var — explicit path (dev) or empty string
 *    (force published mode). Empty string means "do not mount source".
 * 2. Walk up from the package directory looking for apps/celilo/. Works
 *    automatically when @celilo/e2e is used directly inside the infra repo.
 * 3. undefined — falls back to the baked-in npm package inside Docker.
 */
function findCeliloRoot(): string | undefined {
  const envRoot = process.env.CELILO_E2E_INFRA_ROOT;
  if (envRoot !== undefined) {
    return envRoot === '' ? undefined : envRoot;
  }

  let dir = PACKAGE_ROOT;
  for (let i = 0; i < 10; i++) {
    if (existsSync(join(dir, 'apps', 'celilo'))) {
      return dir;
    }
    const parent = resolve(dir, '..');
    if (parent === dir) break;
    dir = parent;
  }
  return undefined;
}

const COMPOSE_TIMEOUT = 120_000;

// Track active project names so process-level cleanup can tear them down.
const activeProjects: Set<string> = new Set();

/**
 * Process-level cleanup: tear down any active e2e resources when the
 * process exits — whether from a clean exit, SIGTERM, SIGINT, or an
 * unhandled error. This catches the case where a test times out and
 * afterAll doesn't run.
 *
 * NOTE: only tears down per-test projects. Shared infra is managed
 * separately (by the test runner or explicit stopSharedInfra call).
 *
 * MUST stay fully synchronous: it runs from an `exit` handler, where the
 * event loop is already closed and any async work is silently dropped.
 */
/**
 * The commands that remove a per-test project WITHOUT needing its compose file.
 * Pure (Rule 10.1) so the no-compose-dependency property is testable.
 *
 * `docker compose -p <project> down` — what this used to run — resolves the
 * project from a compose file in the CURRENT DIRECTORY. An exit handler has no
 * reliable cwd, so compose exited "no configuration file provided", the error
 * was swallowed by the surrounding catch, and nothing was removed. The handler
 * fired correctly and tore down nothing.
 *
 * Removing by name needs only the project name, which we already have.
 * Per-test projects are `celilo-e2e-<timestamp>`; the shared project is
 * `celilo-e2e-shared`, which contains no timestamp and so is never matched —
 * shared infra stays up, as it must.
 */
export function projectTeardownCommands(project: string): {
  listContainers: string;
  /** Sim-created LXC guests of this project — by label, since their names carry no timestamp. */
  listGuests: string;
  listNetworks: string;
  listVolumes: string;
} {
  return {
    listContainers: `docker ps -aq --filter name=${project}`,
    listGuests: `docker ps -aq --filter label=${GUEST_PROJECT_LABEL}=${project}`,
    listNetworks: `docker network ls --format {{.Name}} --filter name=${project}`,
    // Volumes were missing here, and the compose path that would have removed
    // them (`down --volumes`) is the same best-effort call that silently does
    // nothing without a resolvable compose file. So every run leaked its
    // volumes: 58 `celilo-e2e-<ts>_ssh-keys` volumes were found on the builder,
    // the oldest months old. Nothing breaks from a leaked volume the way a
    // leaked network breaks routing — it just accumulates unbounded, and the
    // pile is the evidence that teardown has not been running.
    listVolumes: `docker volume ls -q --filter name=${project}`,
  };
}

/**
 * Command listing the containers attached to one of the project's networks.
 *
 * A Proxmox guest is named `celilo-e2e-lxc-<vmid>` and carries NO project name
 * (it is created by `docker run`, not compose), so the project-scoped container
 * sweep never sees it. A live guest keeps `docker network rm` failing forever,
 * and the leaked network holds the sim's subnet — every later run then dies at
 * `docker compose up` with `invalid pool request: Pool overlaps` (ce-ywix:
 * three full regressions lost to one crashed suite's guest).
 */
export function networkContainersInspectCommand(network: string): string {
  return `docker network inspect ${network} --format {{json .Containers}}`;
}

/**
 * One best-effort cleanup step that failed and was carried past. A sweep step
 * never aborts its sweep (the next step's resources are no less leaked for
 * it), but it is also never silent: each failure is logged as a
 * `[cleanup:failed]` line and counted in the sweep's return value, so the
 * progress line cannot claim "cleanup complete" over a sweep that did not
 * complete (ce-yuzi, lane A2 of openspec/changes/e2e-suite-recovery).
 */
export interface CleanupFailure {
  /** The step that failed, named the way the sweep runs it, e.g. `docker rm -f ...`. */
  step: string;
  /** The error's own message, not a restatement of it. */
  detail: string;
}

/** Log one best-effort step that failed where there is no sweep to count it. */
function logCleanupFailure(step: string, error: unknown): void {
  const detail = error instanceof Error ? error.message : String(error);
  console.error(`[cleanup:failed] ${step} | ${detail}`);
}

/** Record a carried-past failure: log it AND count it, so the caller's
 * progress line reflects reality. */
function recordFailure(failures: CleanupFailure[], step: string, error: unknown): void {
  failures.push({ step, detail: error instanceof Error ? error.message : String(error) });
  logCleanupFailure(step, error);
}

/**
 * The done-side of the stale-resource sweep's progress line. Honest by
 * construction: "complete" appears only when the sweep returned zero
 * failures.
 */
export function cleanupProgress(failures: CleanupFailure[]): string {
  if (failures.length === 0) return 'cleanup complete';
  return `cleanup incomplete: ${failures.length} step(s) failed (see [cleanup:failed] lines above)`;
}

/**
 * Force-remove a project's containers, networks and volumes by name. Never
 * throws. Returns one CleanupFailure per step that failed and was carried
 * past, so callers can report what the sweep could not do.
 */
export function forceRemoveProject(project: string): CleanupFailure[] {
  const failures: CleanupFailure[] = [];
  const cmds = projectTeardownCommands(project);
  const listed = (cmd: string, step: string): string[] => {
    try {
      return run(cmd, { timeout: 15_000, stdio: 'pipe' }).split('\n').filter(Boolean);
    } catch (err) {
      // A listing that fails must not look like an empty result: an empty
      // answer and a broken docker are indistinguishable to the sweep, and
      // the sweep would skip what it cannot see.
      recordFailure(failures, step, err);
      return [];
    }
  };
  // Guests are listed by LABEL, not only by name: a sim-created guest is named
  // celilo-e2e-lxc-<vmid>, which the name filter cannot match, and a guest left
  // running holds its zone network's endpoint — the network rm below then fails
  // with "has active endpoints" and the subnet stays allocated, colliding with
  // every later suite in the run (celilo#1247).
  const ids = [
    ...new Set([
      ...listed(cmds.listContainers, `docker ps name=${project}`),
      ...listed(cmds.listGuests, `docker ps label=${GUEST_PROJECT_LABEL}=${project}`),
    ]),
  ];
  if (ids.length > 0) {
    try {
      run(`docker rm -f ${ids.join(' ')}`, { timeout: 30_000, stdio: 'pipe' });
    } catch (err) {
      recordFailure(failures, `docker rm -f ${project} containers`, err);
    }
  }
  try {
    const nets = run(cmds.listNetworks, { timeout: 15_000, stdio: 'pipe' })
      .split('\n')
      .filter(Boolean);
    for (const net of nets) {
      // Remove the e2e containers ON the network first, or the rm below fails
      // while a guest is attached. Only ever touches containers carrying the
      // provisioner's own CONTAINER_PREFIX containment contract, so anything
      // else attached by hand is left alone and the network rm fails (the
      // doctor's leaked-stacks check is what names that case).
      try {
        // Through the seam (run/dockerCli), not raw execSync, so the sweep is
        // unit-testable and every failure lands in the same channel.
        const attached = JSON.parse(
          run(networkContainersInspectCommand(net), { timeout: 15_000, stdio: 'pipe' }) || '{}',
        ) as Record<string, { Name?: string }>;
        const ours = Object.values(attached)
          .map((c) => c.Name ?? '')
          .filter((n) => n.startsWith(CONTAINER_PREFIX));
        if (ours.length > 0) {
          run(`docker rm -f ${ours.join(' ')}`, { timeout: 30_000, stdio: 'pipe' });
        }
      } catch (err) {
        recordFailure(failures, `attached-container sweep on ${net}`, err);
      }
      try {
        run(`docker network rm ${net}`, { timeout: 10_000, stdio: 'pipe' });
      } catch (err) {
        // A network that fails to rm must not stop the volume sweep below.
        recordFailure(failures, `docker network rm ${net}`, err);
      }
    }
  } catch (err) {
    recordFailure(failures, `network sweep for ${project}`, err);
  }
  // Volumes last: a volume still attached to a container cannot be removed, so
  // this must follow the container sweep above.
  try {
    const vols = run(cmds.listVolumes, { timeout: 15_000, stdio: 'pipe' })
      .split('\n')
      .filter(Boolean);
    for (const vol of vols) {
      try {
        run(`docker volume rm ${vol}`, { timeout: 10_000, stdio: 'pipe' });
      } catch (err) {
        recordFailure(failures, `docker volume rm ${vol}`, err);
      }
    }
  } catch (err) {
    recordFailure(failures, `volume sweep for ${project}`, err);
  }
  return failures;
}

/**
 * The start-of-run stale-resource sweep: per-test containers, sim-created
 * guests of ANY project (their names carry no timestamp, celilo#1247), whole
 * per-test projects found via their networks, and stale volumes. Moved
 * verbatim out of startNetwork so its reach and its failure behavior are
 * unit-testable (ce-yuzi). Returns the failures it carried past; the caller
 * turns that into the progress line.
 */
export function sweepStaleTestResources(): CleanupFailure[] {
  const failures: CleanupFailure[] = [];
  // Force-kill per-test containers (not shared infra)
  try {
    const containers = run('docker ps -aq --filter "name=celilo-e2e-1" 2>/dev/null');
    if (containers.trim()) {
      run(`docker rm -f ${containers.replace(/\n/g, ' ')}`, { timeout: 30_000 });
    }
  } catch (err) {
    recordFailure(failures, 'stale container sweep', err);
  }

  // Sim-created LXC guests. The name filter above cannot match them (their
  // names are celilo-e2e-lxc-<vmid>, with no timestamp), and a guest left
  // running holds its zone network's endpoint — so the network sweep below
  // cannot free that subnet either, and every later suite in the run dies
  // creating it (celilo#1247). Any container carrying the guest label is
  // ours by construction: only the provisioner sets it.
  try {
    const guests = run(`docker ps -aq --filter label=${GUEST_PROJECT_LABEL}`);
    if (guests.trim()) {
      run(`docker rm -f ${guests.replace(/\n/g, ' ')}`, { timeout: 30_000 });
    }
  } catch (err) {
    recordFailure(failures, 'stale guest sweep', err);
  }

  // Remove stale per-test networks AND their volumes, via the same
  // by-name teardown the exit handler uses. This used to lean on
  // `docker compose -p <project> down --volumes` for the volume half, which
  // has no `-f` and no reliable cwd here either — compose exits "no
  // configuration file provided" and the catch swallows it, so volumes were
  // never removed on this path either. Reusing forceRemoveProject keeps the
  // crash-recovery sweep and the exit handler from drifting apart.
  try {
    const networks = run('docker network ls --format "{{.Name}}" 2>/dev/null')
      .split('\n')
      .filter((n) => n.startsWith('celilo-e2e-1')); // per-test projects start with timestamp
    const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
    for (const project of projects) {
      failures.push(...forceRemoveProject(project));
    }
  } catch (err) {
    recordFailure(failures, 'stale project sweep', err);
  }
  // Volumes can outlive every container and network of their project — that
  // is exactly the 58-volume pile — so sweep by prefix too, not only for
  // projects that still have a network to be discovered by.
  try {
    const vols = run('docker volume ls -q --filter name=celilo-e2e-1 2>/dev/null')
      .split('\n')
      .filter(Boolean);
    for (const vol of vols) {
      try {
        run(`docker volume rm ${vol}`, { timeout: 5_000 });
      } catch (err) {
        recordFailure(failures, `docker volume rm ${vol}`, err);
      }
    }
  } catch (err) {
    recordFailure(failures, 'stale volume sweep', err);
  }
  return failures;
}

function cleanupOnExit() {
  // `--keep` / `--reuse` mean the operator wants the stack to survive for
  // debugging — and they want it MOST after a failure, which is exactly the
  // path this handler covers. Tearing down here would delete the evidence.
  if (process.env.CELILO_E2E_KEEP === '1' || process.env.CELILO_E2E_REUSE === '1') return;

  for (const project of activeProjects) {
    // Compose first when it can work — it also drops volumes and orphans — but
    // it is best-effort, so never rely on it having done anything.
    try {
      run(`docker compose -f ${COMPOSE_FILE} -p ${project} down --volumes --remove-orphans`, {
        cwd: PACKAGE_ROOT,
        timeout: 30_000,
        stdio: 'pipe',
      });
    } catch (err) {
      // Best effort: an exit handler has no reliable cwd, so compose can fail
      // to find the file. forceRemoveProject below is the authoritative,
      // compose-free teardown, but the compose attempt still gets logged so a
      // systematic failure is visible instead of absorbed.
      logCleanupFailure(`compose down ${project}`, err);
    }
    // Authoritative: needs no compose file, no cwd, no working directory state.
    forceRemoveProject(project);
  }
  activeProjects.clear();
  try {
    run('docker network prune -f', { timeout: 10_000, stdio: 'pipe' });
  } catch (err) {
    logCleanupFailure('docker network prune', err);
  }
}

process.on('SIGTERM', () => {
  cleanupOnExit();
  process.exit(1);
});
process.on('SIGINT', () => {
  cleanupOnExit();
  process.exit(1);
});
// `exit`, NOT `beforeExit`: the runner always terminates via `process.exit()`
// (runner.ts ends with `process.exit(failed > 0 ? 1 : 0)`), and `beforeExit`
// does not fire for an explicit exit — nor after an uncaught throw. So on the
// FAILURE path, where the test never reaches `handle.stop()`, teardown was
// never running and the per-test networks leaked.
//
// That leak used not to be confined to the run that caused it. The sim's
// networks were numbered exactly like a real celilo fleet's zones, so on a host
// that lives in those subnets — celilo's own forgejo-builder does — a leaked
// bridge left a DUPLICATE route for a production prefix, and
// traffic from any later container to the real host at that address was routed
// into the dead simulated network and blackholed. That took out three
// consecutive release runs, each dying in `git fetch` after a ~132s connect
// timeout, long after the e2e run that stranded the network had ended.
//
// The sim now lives in SIM_PRIVATE_SUPERNET (#539), so a leak can no longer
// collide with a fleet address — but a leak is still a leak. The start-of-run
// prefix sweep stays as the crash-recovery backstop (it is the only thing that
// can clean up after SIGKILL); this makes the common case clean up after itself
// instead of leaving containers and volumes behind. See #508.
process.on('exit', () => {
  cleanupOnExit();
});

function run(cmd: string, opts?: ExecSyncOptions): string {
  // Encoding is pinned last so TS narrows the return to `string` even if
  // a caller passes an opts object that could in principle override it.
  return dockerCli
    .exec(cmd, {
      timeout: COMPOSE_TIMEOUT,
      stdio: ['pipe', 'pipe', 'pipe'],
      ...opts,
      encoding: 'utf-8',
    })
    .trim();
}

const COMPOSE_FILE = 'docker-compose.test.yml';
const SHARED_COMPOSE_FILE = 'docker-compose.shared.yml';

/**
 * Containers that live in the shared infra project, not the per-test project.
 *
 * CAVEAT: these names are matched globally — a per-test target-machine with
 * any name in this set will be silently routed to the shared-infra container
 * instead of its own. If you're adding a new e2e test machine, pick a name
 * NOT in this set (e.g., use 'celilo-registry' for an LXC hosting the
 * registry module rather than 'registry', which matches the shared stand-in).
 */
const SHARED_CONTAINERS = new Set([
  'root-dns',
  'tld-dns',
  'namecheap-dns',
  'letsencrypt',
  'registry',
  'isitup',
]);

/**
 * Reset the SHARED authoritative DNS (namecheap-dns) to its pristine seed
 * before a test starts, so a prior test's DDNS writes don't bleed forward.
 *
 * namecheap-dns is SHARED across the whole suite. Knot serves the WRITABLE
 * `/config/<zone>.zone` copies, which the DDNS simulator mutates at runtime;
 * `/seed/<zone>.zone` is the read-only pristine mount, copied to `/config` once
 * by namecheap-startup.sh at container start. So the only correct reset is to
 * re-copy `/seed -> /config` inside the container and reload Knot.
 *
 * History: the previous implementation rewrote the HOST-side zone file (which
 * is bind-mounted to `/seed`, not `/config`) and reloaded — Knot never re-read
 * it, so the scrub was a silent no-op. iamtheinternet.org survived only because
 * every test re-registers its own hosts; celilo.computer's apex is never
 * re-registered, so once a celilo.computer-managed test clobbered it (apex ->
 * the fw-ext source IP instead of the 100.64.0.58 website-sim) it stayed broken
 * for the rest of the run — silently failing install-sh. (ISS: cele2e DNS bleed.)
 *
 * Called automatically by startNetwork unless config.skipDnsScrub is set
 * (useful for --keep debugging).
 *
 * Exported for the effect-proving integration test (e2e-confidence #256).
 */
export async function scrubDnsZones(): Promise<void> {
  const exec = (cmd: string): string =>
    run(
      `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} exec -T namecheap-dns sh -c ${JSON.stringify(
        cmd,
      )}`,
      { cwd: PACKAGE_ROOT },
    );

  try {
    exec('cp -f /seed/*.zone /config/ && knotc zone-reload');
  } catch (err) {
    // Non-fatal hygiene measure: log and continue. A failed reset just means a
    // test may see stale DNS — better than aborting the run.
    console.warn(`[dns-scrub] Failed to reset zones to seed: ${err}`);
    return;
  }

  // Verify the reset ACTUALLY took effect by querying the RUNNING authoritative
  // server (not a file): the old scrub silently wrote /seed while Knot served
  // /config, so a file check would have looked fine while the server served
  // stale records. Assert celilo.computer's apex now resolves to the website-sim
  // and throw loudly on a confirmed mismatch, so a broken reset can never again
  // silently bleed DNS state across tests. (e2e-confidence #253.)
  const expected = SIMULATOR_IPS.WEBSITE;
  let served: string | null = null;
  try {
    served = exec('kdig @127.0.0.1 celilo.computer A +short').trim();
  } catch (err) {
    // Couldn't run the verification query (e.g. the query tool is unavailable).
    // Warn rather than abort the suite over a missing diagnostic — we only fail
    // on a CONFIRMED wrong answer below.
    console.warn(`[dns-scrub] could not verify reset (DNS query failed): ${err}`);
  }
  if (served !== null && !served.split(/\s+/).includes(expected)) {
    throw new Error(
      `[dns-scrub] post-reset verification FAILED: celilo.computer apex should serve ${expected} (website-sim) after scrub, but namecheap-dns returns "${served || '(empty)'}". The DNS reset did not take effect — shared DNS would bleed across tests.`,
    );
  }
}

/** Exported for the unit tests that drive it through a fake DockerCli. */
export function dockerExec(
  projectName: string,
  composeDir: string,
  container: string,
  cmd: string,
  timeoutMs = 60_000,
): ExecResult {
  // Use the shared infra project for shared containers
  const isShared = SHARED_CONTAINERS.has(container);
  const project = isShared ? SHARED_PROJECT_NAME : projectName;
  const composeFile = isShared ? SHARED_COMPOSE_FILE : COMPOSE_FILE;

  try {
    const stdout = run(
      `docker compose -f ${composeFile} -p ${project} exec -T ${container} bash -c ${JSON.stringify(cmd)}`,
      { cwd: composeDir, timeout: timeoutMs },
    );
    return { stdout, stderr: '', exitCode: 0 };
  } catch (err: unknown) {
    const e = err as {
      stdout?: string;
      stderr?: string;
      status?: number;
      code?: string;
      signal?: string;
      killed?: boolean;
    };
    // execSync's timeout surfaces as an opaque error (ETIMEDOUT / SIGTERM /
    // Bun's "canceled") with empty stderr — the classic e2e time-sink where a
    // hung command reads as a mystery. Replace it with what actually happened.
    const timedOut = e.code === 'ETIMEDOUT' || e.signal === 'SIGTERM' || e.killed === true;
    const stderr = timedOut
      ? `timed out after ${Math.round(timeoutMs / 1000)}s running: ${cmd}`
      : (e.stderr?.toString() ?? '');
    return {
      stdout: e.stdout?.toString() ?? '',
      stderr,
      // The timeout verdict wins over any status the dying client left behind.
      // `docker compose exec` traps the SIGTERM execSync sends on timeout and
      // exits 130 by its own convention (ce-013r, celilo#1293), so the real
      // error carries status: 130 — the client's death code, not the command's
      // verdict. Adopting it read every harness timeout as a mystery exit 130.
      exitCode: timedOut ? 124 : (e.status ?? 1),
    };
  }
}

/**
 * Subnets the simulated internet actually hosts. A nameserver outside all of
 * them blackholes: nothing in the sim owns the address, so every query to it
 * eats its full timeout. ZONE_SUBNETS covers machines a test deploys (a
 * dns_internal provider among them); the three external networks cover the
 * sim's public edge — isp-external (comcast-resolver, the fleet's
 * `dns.primary`), internet-external (the public simulators), and the
 * real-internet transit network behind fw-ext.
 */
const SIM_ROUTABLE_SUBNETS: ReadonlyArray<{ cidr: string; label: string }> = [
  ...Object.entries(ZONE_SUBNETS).map(([zone, cidr]) => ({ cidr, label: `${zone} zone` })),
  { cidr: '203.0.113.0/24', label: 'isp-external' },
  { cidr: '100.64.0.0/24', label: 'internet-external' },
  { cidr: '172.30.0.0/24', label: 'real-internet' },
];

/**
 * The nameserver problems in `celilo system config get` output, one string
 * each. Pure so the gate is unit-testable without a daemon. `dns.fallback` is
 * comma-separated; `dns.primary` a single IP — both split on either separator.
 * A missing `dns.primary` is itself a problem: the birth nameserver list would
 * be empty.
 */
export function fleetNameserverProblems(configOutput: string): string[] {
  const entries = [...configOutput.matchAll(/^dns\.(primary|fallback) = (.+)$/gm)].map(
    ([, key, value]) => ({ key: key as 'primary' | 'fallback', value: value.trim() }),
  );
  const problems: string[] = [];
  if (!entries.some((e) => e.key === 'primary')) {
    problems.push('dns.primary is not set — the birth nameserver list would be empty');
  }
  for (const { key, value } of entries) {
    for (const ip of value.split(/[\s,]+/).filter(Boolean)) {
      if (parseIpv4(ip) === null) {
        problems.push(`dns.${key} entry "${ip}" is not an IPv4 address`);
        continue;
      }
      const routable = SIM_ROUTABLE_SUBNETS.some((s) => subnetContains(s.cidr, ip));
      if (!routable) {
        problems.push(
          `dns.${key} nameserver ${ip} is in no simulated subnet (${SIM_ROUTABLE_SUBNETS.map((s) => s.cidr).join(', ')}) — the sim cannot route to it, so every lookup to it blackholes`,
        );
      }
    }
  }
  return problems;
}

/**
 * Recurrence gate for the Gathering Facts hang (celilo#1290). A nameserver in
 * fleet config that the sim cannot route blackholes exactly the lookups
 * ansible's Gathering Facts performs, and the failure used to surface minutes
 * later as an unrelated 600s command timeout. Read back what `system init`
 * actually stored — core's own resolver discovery falls back to 1.1.1.1
 * (dns-discovery.ts), which is unroutable here — and fail in seconds, naming
 * the address.
 */
function assertFleetNameserversAreSimRoutable(projectName: string, composeDir: string): void {
  const result = dockerExec(
    projectName,
    composeDir,
    'management',
    'celilo system config get dns.primary; celilo system config get dns.fallback',
    30_000,
  );
  // A missing dns.fallback exits non-zero ("key not found"); that is fine and
  // expected. Only treat the read as failed when even dns.primary is absent.
  if (result.exitCode !== 0 && !result.stdout.includes('dns.primary = ')) {
    throw new Error(
      `Could not read fleet DNS config from the management container: ${result.stderr || result.stdout || '(no output)'}`,
    );
  }
  const problems = fleetNameserverProblems(result.stdout);
  if (problems.length > 0) {
    throw new Error(
      `Fleet DNS config names nameserver(s) the simulated internet cannot reach — the celilo#1290 failure shape (600s Gathering Facts hang):\n  ${problems.join('\n  ')}`,
    );
  }
}

/**
 * Fail fast if the management image's baked celilo CLI is older than the
 * harness needs (ce-5qp). Runs before `system init` so a version mismatch
 * surfaces in <10s with an actionable message instead of a 90s hang ending
 * in "canceled".
 */
function assertCliVersion(projectName: string, composeDir: string): void {
  const result = dockerExec(projectName, composeDir, 'management', 'celilo --version', 8_000);
  if (result.exitCode !== 0) {
    throw new Error(
      `Could not read celilo version from the management image (harness requires >=${MIN_CLI_VERSION}). \`celilo --version\` exited ${result.exitCode}: ${result.stderr || result.stdout || '(no output)'}`,
    );
  }
  const problem = checkCliVersion(result.stdout);
  if (problem) throw new Error(problem);
}

/**
 * Exec into a container that is NOT part of the compose project.
 *
 * A guest the Proxmox simulator provisioned is a real container, but compose
 * knows nothing about it, so `docker compose exec` cannot see it. Same wrapping
 * as `dockerExec` so both behave identically from a test's point of view.
 */
export function plainDockerExec(container: string, cmd: string, timeoutMs = 60_000): ExecResult {
  try {
    const stdout = run(`docker exec ${container} bash -c ${JSON.stringify(cmd)}`, {
      timeout: timeoutMs,
    });
    return { stdout, stderr: '', exitCode: 0 };
  } catch (err: unknown) {
    const e = err as {
      stdout?: string;
      stderr?: string;
      status?: number;
      code?: string;
      signal?: string;
      killed?: boolean;
    };
    // Same timeout mapping as dockerExec: the harness's clock ended the exec,
    // so it reports 124 with the actionable stderr, never the client's own
    // exit status (ce-013r).
    const timedOut = e.code === 'ETIMEDOUT' || e.signal === 'SIGTERM' || e.killed === true;
    return {
      stdout: e.stdout ?? '',
      stderr: timedOut
        ? `timed out after ${Math.round(timeoutMs / 1000)}s running: ${cmd}`
        : (e.stderr ?? String(err)),
      exitCode: timedOut ? 124 : (e.status ?? 1),
    };
  }
}

/**
 * Finds the interface still carrying an address in `subnet` (the alien
 * segment's subnet) inside container `service`, deletes it, and verifies the
 * deletion by re-reading the interface table. Throws naming the surviving
 * interface and container when the interface is still there — the detach path
 * of attachAlienSegment, which re-reads and throws for the same reason:
 * neither docker's exit code nor its own view can be believed for veth
 * surgery, and a surviving alien interface is exactly what makes a later
 * converge refuse somewhere far from the cause (celilo#1261).
 *
 * `exec` is injected so unit tests can drive this against a faked interface
 * table with no docker at all.
 */
export function removeAlienInterface(opts: {
  service: string;
  subnet: string;
  exec: (cmd: string) => ExecResult;
  /** Captured failure of `docker network disconnect -f`, carried into the
   * throw's report when the repair does not hold. It is evidence, not a
   * verdict: disconnect legitimately fails on a container docker already
   * considers unattached. */
  disconnectError?: string;
}): void {
  const prefix = opts.subnet.replace(/\.\d+\/\d+$/, '').replace(/\./g, '\\.');
  const orphanRegex = new RegExp(`(\\S+)\\s+inet\\s+${prefix}\\.\\d+`);
  const orphanIn = (table: string) => orphanRegex.exec(table)?.[1];

  const check = opts.exec('ip -o addr show scope global');
  const orphan = orphanIn(check.stdout);
  if (!orphan) return;

  const del = opts.exec(`ip link del ${orphan}`);
  const recheck = opts.exec('ip -o addr show scope global');
  const survivor = orphanIn(recheck.stdout);
  if (survivor) {
    throw new Error(
      [
        `detachAlienSegment: '${opts.service}' still has interface ${survivor} on ${opts.subnet} after repair:`,
        `  ip link del ${orphan} exited ${del.exitCode}, stderr: ${del.stderr || '(empty)'}`,
        opts.disconnectError
          ? `  docker network disconnect failed: ${opts.disconnectError}`
          : undefined,
        `  interface table after repair:\n${recheck.stdout}`,
      ]
        .filter((line) => line !== undefined)
        .join('\n'),
    );
  }
  // The delete reported failure but the interface is gone. docker's exit code
  // is not to be believed for this operation (see attachAlienSegment), so this
  // is surfaced rather than swallowed — and not fatal, since the actual goal
  // state (no alien interface) holds.
  if (del.exitCode !== 0) {
    console.error(
      `detachAlienSegment: 'ip link del ${orphan}' on '${opts.service}' exited ${del.exitCode} but the interface is gone. stderr: ${del.stderr || '(empty)'}`,
    );
  }
}

function dockerExecAsync(
  projectName: string,
  composeDir: string,
  container: string,
  cmd: string,
  timeoutMs = 60_000,
): Promise<ExecResult> {
  return new Promise((resolve) => {
    // stdbuf -oL -eL forces line-buffered stdout and stderr so Ansible play/task
    // markers stream in real time rather than arriving in one burst at the end.
    const args = [
      'compose',
      '-f',
      COMPOSE_FILE,
      '-p',
      projectName,
      'exec',
      '-T',
      container,
      'stdbuf',
      '-oL',
      '-eL',
      'bash',
      '-c',
      cmd,
    ];
    const child = dockerCli.spawn(args, { cwd: composeDir });

    let stdout = '';
    let stderr = '';

    child.stdout.on('data', (data: Buffer) => {
      const text = data.toString();
      stdout += text;
      process.stdout.write(text);
    });

    child.stderr.on('data', (data: Buffer) => {
      const text = data.toString();
      stderr += text;
      process.stderr.write(text);
    });

    const timer = setTimeout(() => {
      child.kill('SIGTERM');
      resolve({ stdout, stderr: `${stderr}\ntimeout after ${timeoutMs}ms`, exitCode: 124 });
    }, timeoutMs);

    child.on('close', (code) => {
      clearTimeout(timer);
      resolve({ stdout, stderr, exitCode: code ?? 1 });
    });
  });
}

async function waitFor(
  check: () => Promise<boolean>,
  timeoutMs: number,
  label: string,
  onTimeout?: () => string | Promise<string>,
): Promise<void> {
  const start = Date.now();
  while (Date.now() - start < timeoutMs) {
    try {
      if (await check()) return;
    } catch {
      // retry
    }
    // e2e-sleep-ok: poll cadence inside waitFor; the loop re-checks the condition each iteration.
    await new Promise((r) => setTimeout(r, 2000));
  }
  // Self-diagnosing timeout (e2e-confidence #255): a readiness wait must attach
  // EVIDENCE, never speculate ("X likely stalled or Y is down"). When the caller
  // supplies an onTimeout collector, capture the live state and append it so the
  // failure pinpoints the layer on first occurrence — no hand-instrumentation
  // after the fact.
  let diagnostics = '';
  if (onTimeout) {
    try {
      diagnostics = `\n${await onTimeout()}`;
    } catch (err) {
      diagnostics = `\n(onTimeout diagnostics failed: ${err instanceof Error ? err.message : String(err)})`;
    }
  }
  throw new Error(`Timeout waiting for ${label} after ${timeoutMs}ms${diagnostics}`);
}

/**
 * Which of these image tags docker does not have locally.
 *
 * One `docker images` listing rather than a per-image `inspect`, because this
 * runs on the startup path of every test and forty process spawns there is its
 * own cost. A docker failure returns "all of them", so the caller builds —
 * being wrong in the direction of doing the work.
 */
/** Exported for the unit tests that drive it through a fake DockerCli. */
export function missingImages(tags: string[]): string[] {
  if (tags.length === 0) return [];
  let present: Set<string>;
  try {
    present = new Set(
      run('docker images --format "{{.Repository}}:{{.Tag}}"', { timeout: 15_000 })
        .split('\n')
        .map((line) => line.trim())
        .filter(Boolean),
    );
  } catch {
    return tags;
  }
  return tags.filter((tag) => !present.has(tag.includes(':') ? tag : `${tag}:latest`));
}

/**
 * Build the NetworkHandle shared by startNetwork (fresh network) and
 * reconnectNetwork (--keep reuse). Both expose the identical method set;
 * only the setup that precedes handle creation differs, so the handle
 * methods live here once instead of being duplicated in both functions.
 */
function buildNetworkHandle(
  projectName: string,
  composeDir: string,
  celiloRoot: string | undefined,
  topology: TopologyPreset,
): NetworkHandle {
  // Track host-side resources spawned via the handle (SOCKS proxies,
  // Playwright browsers). stop() tears these down before the docker
  // network itself.
  const activeProxies: SocksProxyHandle[] = [];
  /** Services plugged into the undeclared segment, and the subnet it uses. */
  const attachedToAlienSegment = new Set<string>();
  let alienSegmentSubnet = '';
  const activeBrowsers: BrowserHandle[] = [];

  const handle: NetworkHandle = {
    projectName,

    async celilo(
      cmd: string,
      optsOrTimeout?: number | { check?: boolean; timeoutMs?: number },
    ): Promise<ExecResult> {
      const check = typeof optsOrTimeout === 'object' ? (optsOrTimeout.check ?? true) : true;
      const timeoutMs =
        typeof optsOrTimeout === 'number' ? optsOrTimeout : (optsOrTimeout?.timeoutMs ?? 120_000);

      const result = await dockerExecAsync(
        projectName,
        composeDir,
        'management',
        `celilo ${cmd}`,
        timeoutMs,
      );

      if (check && result.exitCode !== 0) {
        throw new CeliloCommandError(cmd, result);
      }
      return result;
    },

    exec(container: string, cmd: string, timeoutMs = 60_000): Promise<ExecResult> {
      return Promise.resolve(dockerExec(projectName, composeDir, container, cmd, timeoutMs));
    },

    async registerControlPlane(options = {}): Promise<void> {
      const moduleDir =
        options.moduleDir ?? join(celiloRoot ?? PACKAGE_ROOT, 'modules', 'celilo-mgmt');

      await handle.publishModule(moduleDir);
      // Tolerant: this box may already be in the pool depending on how the
      // network came up, and "already there" satisfies the point of the step.
      await handle.celilo('machine add 127.0.0.1 --zone secure-mgmt --earmark celilo-mgmt', {
        check: false,
      });
      await handle.celilo('module import celilo-mgmt');
      // Docker and terraform are already in the management image; installing
      // them again costs minutes and proves nothing here.
      await handle.celilo('module config set celilo-mgmt install_docker false');
      await handle.celilo('module config set celilo-mgmt install_terraform false');
      await handle.celilo(
        'module config set celilo-mgmt db_path /root/.local/share/celilo/celilo.db',
      );
      // Give the box its fleet key BEFORE deploying, using the harness keypair.
      //
      // `ensureFleetKey` (apps/celilo/src/services/fleet-key.ts, reached by the
      // deploy through `celilo system ensure-fleet-key`) mints a new
      // `celilo-fleet` key when none exists and records `ssh.public_key`. In a
      // real fleet that is right. In the rig it is not survivable: every target
      // container trusts the HARNESS key, baked into authorized_keys at boot,
      // and a freshly minted key is trusted by nothing. celilo would hold a key
      // no machine accepts, and the next `machine add` fails with "no matching
      // private key was found in ~/.ssh/" — which reads as a missing fixture and
      // is actually a rotated key.
      //
      // `ensureFleetKey` is idempotent and REUSES an existing key, so seeding it
      // with the harness pair means the deploy runs its real code path and
      // arrives at the key the fleet already trusts. This models an operator
      // whose management box already has its fleet key, which is the ordinary
      // case after the first deploy.
      //
      // The directory below must match `getFleetSshDir()`, which is
      // `dirname(getDbPath())/.ssh`. The `db_path` set just above is what makes
      // the two agree — the framework reads it from the env `on_install`
      // exports before shelling out.
      const stateSshDir = '/root/.local/share/celilo/.ssh';
      dockerExec(
        projectName,
        composeDir,
        'management',
        `mkdir -p ${stateSshDir} && chmod 700 ${stateSshDir} && ` +
          `cp /ssh-keys/id_ed25519 ${stateSshDir}/id_ed25519 && ` +
          `cp /ssh-keys/id_ed25519.pub ${stateSshDir}/id_ed25519.pub && ` +
          `chmod 600 ${stateSshDir}/id_ed25519`,
      );

      await handle.celilo('module deploy celilo-mgmt', 300_000);

      // Deploying celilo-mgmt onto the management box REWRITES that box's own
      // ~/.ssh. A `machine add` landing in that window fails with "Cannot find
      // SSH private key", which reads as a missing fixture rather than as a
      // race against the deploy that just ran. Wait for the key to be back and
      // readable before handing control to the caller, so no suite has to know
      // this happens.
      await waitFor(
        async () => {
          const probe = dockerExec(
            projectName,
            composeDir,
            'management',
            'test -r /root/.ssh/id_ed25519 && echo ok',
          );
          return probe.stdout.trim() === 'ok';
        },
        60_000,
        "the management box's SSH private key to be readable after celilo-mgmt deploy",
      );
    },

    async moduleHost(moduleId: string): Promise<ModuleHost> {
      const status = dockerExec(
        projectName,
        composeDir,
        'management',
        `celilo module status ${moduleId}`,
      );
      const host = parseModuleHost(status.stdout);
      if (!host) {
        throw new Error(
          `Could not find where '${moduleId}' is deployed. Is it deployed yet? \`module status\` reported no placement:\n${status.stdout.slice(0, 400)}`,
        );
      }
      return host;
    },

    async targetIp(moduleId: string): Promise<string> {
      const where = dockerExec(
        projectName,
        composeDir,
        'management',
        `celilo module where ${moduleId} --json`,
      );
      const addresses = parseModuleWhere(where.stdout);
      if (addresses.length === 0) {
        throw new Error(
          `Could not resolve an address for '${moduleId}'. Its deploy did not record one in the inventory\n` +
            `(or the CLI answered something unparsable). Raw:\n${where.stdout.slice(0, 400)}`,
        );
      }
      return addresses[0];
    },

    async execOnModuleHost(moduleId, cmd, timeoutMs = 60_000): Promise<ExecResult> {
      const host = await this.moduleHost(moduleId);
      return host.reach === 'plain'
        ? plainDockerExec(host.container, cmd, timeoutMs)
        : dockerExec(projectName, composeDir, host.container, cmd, timeoutMs);
    },

    async respondWith(values): Promise<void> {
      // Write the values JSON inside the management container at a
      // stable path, then start a detached `celilo events respond
      // --values <path>` process. The responder polls the in-container
      // bus and answers config/secret/ensure events as the deploy
      // emits them. The responder dies with the container at network
      // teardown, so no explicit cleanup hook is needed.
      //
      // Long timeouts: the responder's --idle-timeout has to outlast
      // any quiet stretch between deploys (a long `module deploy`
      // might not emit interview events for several minutes if it's
      // building images). 1h idle / 2h max-duration covers any single
      // test comfortably without leaking. The container goes away at
      // network stop regardless.
      const valuesJson = JSON.stringify(values).replace(/'/g, "'\\''");
      const valuesPath = '/tmp/cele2e-responder-values.json';
      const writeCmd = `printf '%s' '${valuesJson}' > ${valuesPath}`;
      const writeResult = dockerExec(projectName, composeDir, 'management', writeCmd, 10_000);
      if (writeResult.exitCode !== 0) {
        throw new Error(
          `respondWith: failed to write values file inside management container: ${writeResult.stderr}`,
        );
      }
      // Kill any prior responder (idempotent: replaces prior values map).
      //
      // Matched on "events respond", NOT "celilo events respond": inside the
      // e2e management image `celilo` is a dev-mode shim that execs `bun run
      // .../index.ts events respond ...`, so the literal word "celilo" never
      // appears contiguous with "events respond" in the resulting process's
      // argv — the old pattern silently matched nothing, so a second
      // `respondWith()` call left the FIRST responder (and its stale values
      // map) running alongside the new one instead of replacing it.
      dockerExec(
        projectName,
        composeDir,
        'management',
        'pkill -f "events respond" 2>/dev/null || true',
        5_000,
      );
      // Spawn detached. nohup + & + disown cleanly survives the
      // dockerExec session ending.
      const spawnCmd = `nohup celilo events respond --values ${valuesPath} --idle-timeout 1h --max-duration 2h --emittedBy cele2e-responder > /tmp/cele2e-responder.log 2>&1 < /dev/null & disown`;
      const spawnResult = dockerExec(projectName, composeDir, 'management', spawnCmd, 5_000);
      if (spawnResult.exitCode !== 0) {
        throw new Error(
          `respondWith: failed to spawn responder inside management container: ${spawnResult.stderr}`,
        );
      }
      // The spawn returns as soon as the shell backgrounds the process; bun
      // then has to boot inside the container and register the responder's
      // bus watches. A fixed sleep both wastes time on an idle host and is
      // not enough on a loaded one: dns-replication failed stage 1 when its
      // deploy's first interview fired before the responder polled, and the
      // missing-responder error read as a missing fixture value. The log
      // line below is printed only after startProgrammaticResponder has
      // registered every watch, so waiting for it is a readiness wait on the
      // real prerequisite, not a sleep (same pattern as deployFirewall).
      const responderLog = '/tmp/cele2e-responder.log';
      await waitFor(
        async () => {
          const probe = dockerExec(
            projectName,
            composeDir,
            'management',
            `grep -q "programmatic responder running" ${responderLog}`,
            5_000,
          );
          return probe.exitCode === 0;
        },
        60_000,
        'the e2e responder to register its bus watches in the management container',
        async () => {
          const log = dockerExec(
            projectName,
            composeDir,
            'management',
            `cat ${responderLog} 2>/dev/null || echo '(no responder log)'`,
            5_000,
          );
          return `responder log (${responderLog}):\n${log.stdout || log.stderr}`;
        },
      );
    },

    async deployFirewall(opts = {}): Promise<ExecResult> {
      // Everything the firewall declares is derived from the topology it is
      // wired for. `opts.zones` used to select which zones got
      // `provided_networks`, and that cannot survive interface classification:
      // `provided_networks` IS what writes `network.<zone>.subnet`, so a leg
      // left out of it has no declared subnet, classifies `alien`, and the
      // converge refuses on a firewall celilo has never understood.
      //
      // That is the real rule, not a harness quirk. A firewall holding an
      // address on a segment celilo has no subnet for is one celilo cannot
      // describe. So the wired legs and the declared zones are ONE list (D8),
      // and a test wanting fewer legs changes the topology.
      //
      // The old `opts.zones` never said which NICs the box has — the compose
      // topology decides that, and it wires fw-main to every segmented zone
      // regardless of what a test passed. Roughly twenty call sites passed
      // `['dmz']` against a four-leg firewall, so every one of them
      // under-declared the hardware. Deriving from the topology makes that
      // structurally impossible rather than something each test author has to
      // remember, and removes a hand-maintained copy of a list the compose
      // generator already owns.
      //
      // `external` is excluded: it is the residual, has no subnet, and is not a
      // zone modules are placed in.
      //
      // `firewallZoneLegs` knows the topology's legs, but not the control-plane
      // one: the generator adds fw-main's `secure-mgmt` leg as a post-step
      // (when celilo-mgr lives off the LAN, when a machine declares the zone,
      // or when the proxmox sim is on), so it never appears in a
      // topology-derived list. A sim fleet's fw-main then held an address on a
      // segment with no declared subnet, eth4 classified alien, and the
      // iptables converge refused (ce-fvxd). The compose file is the wiring and
      // the wiring decides — the same artifact `topologyFromComposeFile` reads.
      const baseLegs = firewallZoneLegs(topology);
      const declaredZones: FirewallLeg[] = fwMainHasSecureMgmtLeg(join(composeDir, COMPOSE_FILE))
        ? [...baseLegs, 'secure-mgmt']
        : baseLegs;
      const providedZones = declaredZones.filter((zone) => zone !== 'external');
      const firewallIp = ZONE_GATEWAYS.internal; // fw-main on internal
      const natIp = opts.natIp ?? internalNatIp();

      // fw-main is a firewall container, not a target machine, so the
      // network readiness wait (target-setup) does NOT cover its sshd. Poll
      // until the firewall accepts SSH before `machine add` — otherwise a
      // transient first-connect times out (spawnSync ETIMEDOUT) and aborts the
      // whole deploy, cascading into unrelated "module not found" failures
      // (#222). This is a readiness wait on the real prerequisite, not a sleep.
      await waitFor(
        async () => {
          const probe = dockerExec(
            projectName,
            composeDir,
            'management',
            `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 -i /root/.ssh/id_ed25519 root@${firewallIp} hostname`,
            15_000,
          );
          return probe.exitCode === 0;
        },
        60_000,
        `firewall ${firewallIp} sshd`,
      );

      // The addressing of each segmented zone, supplied the way an operator
      // supplies it — `system config set`, celilo's own surface.
      //
      // This used to ride in on `module config set iptables provided_networks`,
      // and the module wrote it to system config from its install hook. Modules
      // no longer define networks
      // (openspec/changes/networks-are-declared-not-written/specs/network-declaration/spec.md);
      // the manifest REQUIRES them, and celilo asks whoever is attached for any
      // it does not already hold. The harness answers in advance rather than
      // through the interview, because these values are the simulator's own
      // address plan (ZONE_SUBNETS) — it is not a stand-in for an operator's
      // judgement, it is the topology this rig physically wired.
      for (const zone of providedZones) {
        await handle.celilo(`system config set network.${zone}.subnet ${ZONE_SUBNETS[zone]}`);
        await handle.celilo(`system config set network.${zone}.gateway ${ZONE_GATEWAYS[zone]}`);
      }

      // When fw-main carries the secure-mgmt leg, the management box sits on the
      // control-plane network behind this firewall, and the firewall must TRUST
      // that subnet or default-DROP drops the SSH `machine add` and every later
      // hook/converge needs (celilo#1353: 19 suites died at machine add with a
      // misleading key-mismatch message; the live stack showed the packet
      // timing out in fw-main's FORWARD chain, policy DROP, trusted sources =
      // internal only).
      //
      // The firewall DERIVES control-plane trust from where the celilo-mgmt
      // module is deployed (apps/celilo/src/hooks/capability-loader.ts
      // loadControlPlaneSubnet), falling back to the internal subnet. Suites
      // that machine-add before any celilo-mgmt deploy have neither, so the
      // declared control-plane subnet is trusted by nothing. The operator in
      // this topology declares it explicitly — `firewall.trusted_subnets` is the
      // product surface for exactly that (composeTrustedSubnets origin:
      // operator-override) — and the harness models that operator rather than
      // widening the product's fallback, which the approved design
      // (openspec/changes/recognize-management-network, D2) deliberately kept
      // as "previous behaviour + report".
      //
      // Idempotent with the derived path: composeTrustedSubnets dedupes by
      // subnet, so a suite that later deploys celilo-mgmt on secure-mgmt renders
      // the same ruleset.
      if (fwMainHasSecureMgmtLeg(join(composeDir, COMPOSE_FILE))) {
        await handle.celilo(
          `system config set firewall.trusted_subnets ${ZONE_SUBNETS['secure-mgmt']}`,
        );
      }

      // fw-main is registered as an internal-zone machine; iptables deploys to it.
      await handle.celilo(
        `machine add ${firewallIp} --ssh-user root --ssh-key-file /root/.ssh/id_ed25519 --zone internal`,
      );
      await handle.celilo('module import iptables');
      await handle.celilo(`module config set iptables firewall_ip ${firewallIp}`);
      await handle.celilo(`module config set iptables nat_ip ${natIp}`);
      await handle.celilo(`module config set iptables zones '${JSON.stringify(declaredZones)}'`);
      // Caller-supplied settings last, so a suite can turn on an opt-in policy
      // the way an operator would rather than assert against the default.
      for (const [key, value] of Object.entries(opts.config ?? {})) {
        await handle.celilo(`module config set iptables ${key} ${value}`);
      }
      // `check: false` returns the failed deploy instead of throwing, so a test
      // can assert on WHAT the refusal said. Interface classification refuses
      // by design (D12 onboarding), and a refusal is only useful if it names
      // the interface — which is an assertion about output, not about an
      // exception having been raised.
      return opts.check === false
        ? await handle.celilo('module deploy iptables', { check: false, timeoutMs: 180_000 })
        : await handle.celilo('module deploy iptables', 180_000);
    },

    async attachAlienSegment(opts: {
      subnet: string;
      containers: string[];
    }): Promise<Record<string, string>> {
      // A REAL cable into a REAL port. Interface classification is about what
      // celilo finds on the box, so a test that fakes the finding — a dummy
      // link, a stubbed interface table — is testing its own fixture. This
      // creates an actual docker network celilo has no subnet for and plugs
      // real containers into it, which is what an operator hanging an
      // unmanaged switch off a spare port produces: a new interface, a global
      // address, and something live on the other side to prove reachability
      // with.
      //
      // The subnet is the caller's, and it must not be one of ZONE_SUBNETS —
      // the whole point is that celilo cannot attribute it.
      const netName = `${projectName}_alien-seg`;
      alienSegmentSubnet = opts.subnet;
      run(`docker network create --driver bridge --subnet ${opts.subnet} ${netName}`, {
        timeout: 20_000,
      });

      const assigned: Record<string, string> = {};
      opts.containers.forEach((service, index) => {
        // .2 upward: .1 is the bridge itself.
        const ip = opts.subnet.replace(/\.0\/\d+$/, `.${index + 2}`);
        const id = run(`docker compose -p ${projectName} -f ${COMPOSE_FILE} ps -q ${service}`, {
          cwd: composeDir,
          timeout: 20_000,
        });
        if (!id) throw new Error(`attachAlienSegment: no container for service '${service}'`);

        // `docker network connect` EXITS NONZERO on a container that already has
        // a default route — "failed to set gateway while updating gateway: file
        // exists" — while attaching the interface perfectly well: the address
        // lands, the veth is up, and traffic flows both ways. Every container in
        // this topology routes through the firewall, so every attach hits it.
        // `docker network inspect` is no help either; it lists no containers for
        // this network whether the attach worked or not.
        //
        // So neither the exit code nor docker's own view can be believed, and
        // the address on the box is the only trustworthy signal. Verified below
        // rather than assumed — a half-completed attach really does leave an
        // address behind with no working wire, which is precisely the state that
        // would make a classification test pass for the wrong reason.
        try {
          run(`docker network connect --ip ${ip} ${netName} ${id}`, { timeout: 20_000 });
        } catch {
          // Ignorable: failure here is the COMMON outcome (the gateway-exists
          // case documented above, hit by every attach), and a connect that
          // genuinely failed cannot hide — the address check below throws.
        }

        const check = dockerExec(projectName, composeDir, service, 'ip -o addr show scope global');
        if (!check.stdout.includes(ip)) {
          throw new Error(
            `attachAlienSegment: '${service}' has no ${ip} after connecting to ${netName}:\n${check.stdout}`,
          );
        }
        assigned[service] = ip;
        attachedToAlienSegment.add(service);
      });
      return assigned;
    },

    async detachAlienSegment(): Promise<void> {
      const netName = `${projectName}_alien-seg`;
      for (const service of attachedToAlienSegment) {
        const id = run(`docker compose -p ${projectName} -f ${COMPOSE_FILE} ps -q ${service}`, {
          cwd: composeDir,
          timeout: 20_000,
        });
        if (!id) continue;
        // `-f` makes disconnect report failure on a container docker considers
        // unattached, which after a half-completed attach is exactly the state
        // being cleaned up here. So the failure is captured for the repair step
        // to carry in its report, not obeyed — whether the interface actually
        // left is decided by the interface table, not by docker's exit code.
        let disconnectError: string | undefined;
        try {
          run(`docker network disconnect -f ${netName} ${id}`, { timeout: 20_000 });
        } catch (err: unknown) {
          disconnectError = err instanceof Error ? err.message : String(err);
        }
        removeAlienInterface({
          service,
          subnet: alienSegmentSubnet,
          disconnectError,
          exec: (cmd) => dockerExec(projectName, composeDir, service, cmd),
        });
      }
      try {
        run(`docker network rm ${netName}`, { timeout: 20_000 });
      } catch (err: unknown) {
        console.error(
          `detachAlienSegment: docker network rm ${netName} failed: ${err instanceof Error ? err.message : String(err)}`,
        );
      }
      attachedToAlienSegment.clear();
    },

    async deployGreenwave(): Promise<void> {
      const routerIp = greenwaveRouterIp();
      // No `machine add` for the router, and do not restore one: greenwave is an
      // appliance module (schema.ts `apiOnly`, ISP router) that talks to it over
      // HTTPS only (scripts/router-api.ts), deploys onto no system
      // (manifest `requires: capabilities: []`, no `system:`), and never enters
      // infrastructure selection. Adding the router as a root machine only ran
      // the fleet aspects against a simulated ISP router over SSH.
      // @psbanka - 2026-09: removed in ce-5b1v; see celilo#1263.
      await handle.celilo('module import greenwave');
      await handle.celilo(`module config set greenwave router_ip ${routerIp}`);
      await handle.celilo('module secret set greenwave router_username admin');
      await handle.celilo('module secret set greenwave router_password admin');
      await handle.celilo('module deploy greenwave', 180_000);
    },

    async socksProxy(options: ProxyOptions = {}): Promise<SocksProxyHandle> {
      const proxy = await startSocksProxy(projectName, options);
      activeProxies.push(proxy);
      return proxy;
    },

    async browser(options: BrowserOptions): Promise<BrowserHandle> {
      const browserHandle = await startBrowser(projectName, options);
      activeBrowsers.push(browserHandle);
      // browser owns its proxy lifecycle; track it here too so stop()
      // doesn't try to stop it twice (idempotent stop() handles this)
      activeProxies.push(browserHandle.proxy);
      return browserHandle;
    },

    async dig(name: string): Promise<string> {
      const result = dockerExec(projectName, composeDir, 'management', `dig +short ${name}`);
      return result.stdout
        .split('\n')
        .filter((l) => !l.startsWith(';;'))
        .join('\n')
        .trim();
    },

    async debug(container = 'management'): Promise<void> {
      // Signal the runner to hand us the terminal
      console.log(`[debug:pause] ${projectName} ${container}`);

      // The runner intercepts [debug:pause] and spawns the interactive
      // shell itself (since it owns the terminal). It writes a signal
      // file when the user exits the shell.
      const signalFile = join(composeDir, `.debug-resume-${process.pid}`);

      // Wait for the runner to create the signal file
      const start = Date.now();
      const timeout = 86_400_000; // 24 hour max debug session
      while (!existsSync(signalFile) && Date.now() - start < timeout) {
        // e2e-sleep-ok: waits on a human exiting the debug shell; the signal file is re-checked above.
        await new Promise((r) => setTimeout(r, 500));
      }

      // Clean up. Ignorable: the file only ends THIS pause's wait loop, its
      // name is pid-unique so a leftover can never satisfy another session's
      // wait, and the next debug pause writes its own.
      try {
        require('node:fs').unlinkSync(signalFile);
      } catch {
        // deliberately ignored, see above
      }

      console.log('[debug:resumed]');
    },

    waitFor,

    async publishModule(localPath: string): Promise<void> {
      const absPath = resolve(localPath);
      const isNetapp = absPath.endsWith('.netapp');
      const moduleId = isNetapp ? basename(absPath).slice(0, -7) : basename(absPath);

      let netappPath: string;
      let cleanup = false;

      if (isNetapp) {
        netappPath = absPath;
      } else {
        const staged = stagedNetappIfCurrent(absPath, moduleId);
        if (staged) {
          // build-infra already produced this, before the stack was competing
          // for the host. Repackaging it here is the work that times out.
          netappPath = staged;
        } else {
          netappPath = join(tmpdir(), `${moduleId}-${Date.now()}.netapp`);
          cleanup = true;
          const celiloCliPath = join(celiloRoot ?? PACKAGE_ROOT, 'apps/celilo/src/cli/index.ts');
          const startedAt = Date.now();
          try {
            // 180s, not 60s: this runs while the stack is up, and the same work
            // that takes 11s on an idle host went past 60s under a live stack.
            run(
              `bun run ${JSON.stringify(celiloCliPath)} package ${JSON.stringify(absPath)} --output ${JSON.stringify(netappPath)}`,
              { timeout: 180_000 },
            );
          } catch (err) {
            const seconds = ((Date.now() - startedAt) / 1000).toFixed(1);
            throw new Error(
              `Failed to package module ${moduleId} at ${absPath} after ${seconds}s (no current staged netapp, so it was rebuilt inside a live stack): ${String(err)}`,
            );
          }
        }
      }

      try {
        // Write HOST-side into the dir bound at the registry's /uploads. The
        // registry rescans that dir on every request, so it needs no HTTP
        // handshake and no restart.
        //
        // Deliberately not a `docker compose cp` into the container: /uploads
        // is a READ-ONLY bind in a consumer install, so the container write
        // failed for every npm-installed consumer while passing forever in the
        // monorepo, where the same path is not a mount at all (celilo#1142).
        // registryUploadsHostDir() is the same function the compose generator
        // builds the mount from, so the two cannot drift apart again.
        const uploadsDir = registryUploadsHostDir();
        mkdirSync(uploadsDir, { recursive: true });
        copyFileSync(netappPath, join(uploadsDir, `${moduleId}.netapp`));
      } finally {
        if (cleanup)
          try {
            execSync(`rm -f ${JSON.stringify(netappPath)}`, { stdio: 'pipe' });
          } catch (err) {
            logCleanupFailure(`rm staged netapp ${netappPath}`, err);
          }
      }
    },

    async configureAcme(): Promise<void> {
      await handle.celilo(
        'module config set caddy acme_ca https://acme-v02.api.letsencrypt.org/dir',
      );
    },

    async stop(): Promise<void> {
      // Tear down browsers/proxies first — they're host-side resources.
      // Each handle's stop() respects CELILO_E2E_KEEP/REUSE itself, so
      // we don't gate this branch on those env vars.
      for (const browser of activeBrowsers) {
        try {
          await browser.close();
        } catch (err) {
          // A browser that refuses to close must not stop the network teardown
          // below — the leaked resource is the network, not the browser.
          logCleanupFailure('browser close', err);
        }
      }
      for (const proxy of activeProxies) {
        try {
          await proxy.stop();
        } catch (err) {
          logCleanupFailure('proxy stop', err);
        }
      }

      // Respect --keep / --reuse: don't tear down the network
      if (process.env.CELILO_E2E_KEEP === '1' || process.env.CELILO_E2E_REUSE === '1') {
        console.log(`[progress:done] network kept alive (project: ${projectName})`);
        activeProjects.delete(projectName); // don't re-teardown on exit
        return;
      }
      console.log('[progress:start] stopping network | network stopped');
      // Kill this project's sim-created guests BEFORE compose down. A guest
      // holds its zone network's endpoint and the ssh-keys volume, so compose
      // down with the guest alive fails both removals, the error is swallowed
      // below, and the subnet stays allocated for every later suite
      // (celilo#1247).
      try {
        const guests = run(projectTeardownCommands(projectName).listGuests);
        if (guests.trim()) {
          run(`docker rm -f ${guests.replace(/\n/g, ' ')}`, { timeout: 30_000 });
        }
      } catch (err) {
        // Compose down below still runs; without the guest gone it fails both
        // removals (celilo#1247), so the failure is worth naming, not absorbing.
        logCleanupFailure(`guest sweep ${projectName}`, err);
      }
      try {
        run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} down --volumes --remove-orphans`, {
          cwd: composeDir,
          timeout: 60_000,
        });
      } catch (err) {
        // Best effort cleanup: stop() must never fail the caller over teardown.
        // The failure is logged so a systematically broken compose is visible.
        logCleanupFailure(`compose down ${projectName}`, err);
      }
      activeProjects.delete(projectName);
    },
  };

  return handle;
}

export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle> {
  // Pre-flight: fail if live containers are running that will compete for resources.
  try {
    const running = run('docker ps --format "{{.Names}}" 2>/dev/null').split('\n').filter(Boolean);
    const heavyPatterns = ['authentik-', 'caddy-', 'build-your-own-internet-'];
    const heavy = running.filter((name) => heavyPatterns.some((p) => name.startsWith(p)));
    if (heavy.length > 0) {
      const names = heavy.map((c) => `   - ${c}`).join('\n');
      throw new Error(
        `\nCannot start e2e tests: ${heavy.length} live container(s) are running:\n${names}\n\nLive and e2e environments are mutually exclusive.\nStop live containers first: cele2e down --all\n`,
      );
    }
  } catch (e) {
    if (e instanceof Error && e.message.includes('Cannot start e2e tests')) throw e;
  }

  // Ensure shared infrastructure (DNS, Pebble, registry, etc.) is running.
  // This is idempotent — if already up, it just verifies health.
  await ensureSharedInfra();

  // Scrub DNS zone files to baseline before each test (H5 in E2E_TEST_UPDATES.md).
  // Prevents DDNS-written records from one test leaking into the next.
  // Tests can opt out with config.skipDnsScrub (e.g., for --reuse debugging).
  if (!config.skipDnsScrub) {
    await scrubDnsZones();
  }

  // Clean up stale per-test containers (self-healing from prior crashes).
  // The sweep is extracted so it is testable; the progress line is honest by
  // construction: "cleanup complete" only when the sweep reported zero
  // failures (ce-yuzi).
  console.log('[progress:start] cleaning up stale test resources | sweeping');
  const cleanupFailures = sweepStaleTestResources();
  console.log(
    `[progress:start] cleaning up stale test resources | ${cleanupProgress(cleanupFailures)}`,
  );

  const projectName = `celilo-e2e-${Date.now()}`;
  activeProjects.add(projectName);

  const composeDir = PACKAGE_ROOT;
  const celiloRoot = config.celiloRoot ?? findCeliloRoot();

  // Generate per-test compose (references shared infra networks as external)
  const yaml = generateTestComposeYaml(config, celiloRoot);
  writeFileSync(join(composeDir, COMPOSE_FILE), yaml);

  // The compose carries `image:` only (never `build:`), so the run-time path
  // cannot build and cannot resolve a FROM from docker.io — the fetch that
  // made 30 outputs fail with `lookup auth.docker.io` (tasks.md 5b.2). A
  // missing baked tag is a loud failure naming the remedy. Checked for BOTH
  // compose files here, before shared infra comes up, so an unpopulated
  // machine is named in seconds rather than discovered mid-`up`.
  const missing = missingImages([
    ...referencedImages(yaml),
    ...referencedImages(generateSharedInfraYaml()),
  ]);
  if (missing.length > 0) {
    throw new Error(
      `${missing.length} baked image(s) missing from the local store, and suite time never builds: ${missing.join(', ')}.\nRemedy: cele2e build-infra (it owns the network phase — docker base images, apt, caddy, docker-ce). The suite run stays hermetic.`,
    );
  }

  console.log('[progress:start] starting containers | containers running');
  run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} up -d`, {
    cwd: composeDir,
    timeout: 120_000,
  });

  // Wait for DNS convergence (comcast-resolver is per-test, needs to reach shared DNS)
  console.log('[progress:start] waiting for DNS convergence | DNS converged');
  await waitFor(
    async () => {
      const result = dockerExec(
        projectName,
        composeDir,
        'comcast-resolver',
        'dig @127.0.0.1 iamtheinternet.org NS +short +timeout=2',
      );
      return result.exitCode === 0 && result.stdout.trim().length > 0;
    },
    60_000,
    'DNS convergence',
  );

  // Wait for internal resolver (split-horizon DNS for management container).
  // Skipped when the test specifies its own machine named `dns-int` — that
  // means the test is replacing the infra resolver with a module under test
  // (e.g., knot-unbound-internal). The bare target machine has no DNS server
  // until its module deploys, so this wait would always time out. Management
  // can still resolve names via the fallback `nameserver 203.0.113.1` in
  // its resolv.conf during deploy.
  //
  // The resolver-under-test may sit in ANY zone, not just `internal`: ISS-0156
  // places the dns_internal provider in a PROTECTED zone (dmz) so it can see
  // protected-zone query sources for split-horizon views. So check every zone's
  // machines for a `dns-int`, not only internalMachines.
  const dnsIntReplaced = [
    ...config.internalMachines,
    ...config.dmzMachines,
    ...config.appMachines,
    ...config.secureMachines,
  ].some((m) => m.name === 'dns-int');
  if (!dnsIntReplaced) {
    console.log('[progress:start] waiting for internal resolver | internal resolver ready');
    await waitFor(
      async () => {
        const result = dockerExec(
          projectName,
          composeDir,
          'dns-int',
          'dig @127.0.0.1 iamtheinternet.org A +short +timeout=2',
        );
        return result.exitCode === 0 && result.stdout.trim().length > 0;
      },
      30_000,
      'internal DNS resolver',
    );
  }

  // Optional routing verification
  if (config.verifyRouting) {
    console.log('[progress:start] verifying routing | routing verified');
    await waitFor(
      async () => {
        const result = dockerExec(
          projectName,
          composeDir,
          'management',
          `ping -c1 -W2 ${SIMULATOR_IPS.ROOT_DNS}`,
        );
        return result.exitCode === 0;
      },
      30_000,
      'routing verification',
    );
  }

  // Wait for all target machines to complete setup (systemd boot + routing).
  // Via getAllMachines so a newly-supported zone cannot be silently skipped —
  // this list was hand-maintained and omitted `secure-mgmt`, so the first
  // machine placed there raced `machine add` and failed on SSH intermittently.
  const allMachines = getAllMachines(config);
  if (allMachines.length > 0) {
    console.log('[progress:start] waiting for target machines | target machines ready');
  }
  for (const machine of allMachines) {
    await waitFor(
      async () => {
        const result = dockerExec(
          projectName,
          composeDir,
          machine.name,
          'systemctl is-active target-setup 2>/dev/null',
        );
        return result.stdout.trim() === 'active';
      },
      // App-zone machines use Dockerfile.target-machine-docker which bakes in
      // dockerd; systemd boot + dockerd init takes ~25-35s, leaving a tight
      // margin against a 30s timeout. 60s covers the observed worst case
      // without slowing down dmz/internal-only tests (which still hit this
      // in ~2-5s).
      60_000,
      `${machine.name} target-setup`,
      // Self-diagnosing via waitFor's onTimeout (e2e-confidence #255). This
      // timeout has twice been read as a simulator or deployment bug when the
      // real cause was inside the unit — target-setup fetching from a slow
      // third party (#560). The unit's own journal says which line it is on.
      () => {
        const dump = (label: string, cmd: string): string => {
          const r = dockerExec(projectName, composeDir, machine.name, cmd);
          return `--- ${label} ---\n${(r.stdout || r.stderr || '(no output)').trim()}`;
        };
        return [
          `=== target-setup diagnostics (${machine.name}) ===`,
          dump('systemctl status', 'systemctl status target-setup --no-pager 2>&1 | head -20'),
          dump('journal', 'journalctl -u target-setup --no-pager 2>&1 | tail -30'),
        ].join('\n');
      },
    );
  }

  // Wait for DHCP client lease.
  //
  // Skipped when the router's DHCP is off, because then NOTHING is serving yet:
  // celilo's own DHCP server is a module, and modules deploy long after the
  // network is up. Waiting here would time out at 60s on every such suite and
  // report it as a network-start failure. Those suites drive the client
  // themselves once their server is deployed.
  if (config.dhcpClient && config.routerDhcp !== false) {
    console.log('[progress:start] waiting for DHCP client lease | DHCP lease acquired');
    // Self-diagnosing via waitFor's onTimeout (e2e-confidence #255): a DHCP-lease
    // timeout is an intermittent in-suite flake (passes solo). On timeout, dump
    // the dhcp-client's interface + dhclient transcript so the failure shows
    // whether DISCOVER/OFFER/REQUEST/ACK completed (server silent) vs. a lease
    // that landed but wasn't recorded.
    await waitFor(
      async () => {
        const result = dockerExec(
          projectName,
          composeDir,
          'dhcp-client',
          'test -f /var/lib/dhcp/dhclient.leases && grep -c lease /var/lib/dhcp/dhclient.leases',
        );
        return result.exitCode === 0 && Number.parseInt(result.stdout.trim()) > 0;
      },
      60_000,
      'DHCP client lease',
      () => {
        const dump = (label: string, cmd: string): string => {
          const r = dockerExec(projectName, composeDir, 'dhcp-client', cmd);
          return `--- ${label} ---\n${(r.stdout || r.stderr || '(no output)').trim()}`;
        };
        return [
          '=== DHCP diagnostics (dhcp-client) ===',
          dump('ip addr', 'ip -o addr show 2>&1'),
          dump('dhclient.leases', 'cat /var/lib/dhcp/dhclient.leases 2>&1 | tail -25'),
          dump(
            'dhclient transcript',
            "journalctl -u dhclient --no-pager 2>/dev/null | tail -25 || cat /var/log/dhclient.log 2>/dev/null | tail -25 || echo '(no dhclient log)'",
          ),
        ].join('\n');
      },
    );
  }

  // Wait for routing to Pebble (through shared infra networks)
  if (allMachines.length > 0) {
    await waitFor(
      async () => {
        const result = dockerExec(
          projectName,
          composeDir,
          'management',
          `ping -c1 -W2 ${SIMULATOR_IPS.PEBBLE}`,
        );
        return result.exitCode === 0;
      },
      30_000,
      'routing to Pebble',
    );
  }

  // Initialize celilo. Skipped for the vanilla management variant —
  // there's no celilo binary baked into that image; the caller is
  // responsible for installing it (typically by running install.sh)
  // and then calling `celilo system init` themselves.
  if (config.managementVariant !== 'vanilla') {
    assertCliVersion(projectName, composeDir);
    console.log('[progress:start] initializing celilo | celilo initialized');
    // dns.fallback is deliberately NOT set. The sim hosts exactly one resolver
    // the fleet may name — comcast-resolver at 203.0.113.1 — so the
    // pre-resolver birth list is just that address. The former value
    // (1.0.0.1,8.8.8.8) named addresses that exist nowhere in the sim: every
    // lookup to them ate its full timeout, and ansible's Gathering Facts does
    // reverse lookups, which is the 600s hang in celilo#1290. The sim's
    // second public resolver (SIMULATOR_IPS.PUBLIC_RESOLVER) stays OUT of
    // fleet config on purpose: the public_dns check needs a resolver celilo
    // does not itself use.
    // Honest harness (openspec/specs/progressive-zone-disclosure/spec.md): seed ONLY the
    // `internal` zone — the network the management box is genuinely on —
    // plus DNS. dmz/app/secure are NOT pre-seeded; they come into being
    // when a test deploys the firewall (net.deployFirewall()), exactly as
    // in production. A test that places services in those zones must call
    // deployFirewall first; internal-only tests need nothing more.
    const initResult = dockerExec(
      projectName,
      composeDir,
      'management',
      `celilo system init --accept-defaults \
        network.internal.subnet=${ZONE_SUBNETS.internal} \
        network.internal.gateway=${ZONE_GATEWAYS.internal} \
        ${
          // The control plane needs its subnet declared if celilo-mgr lives
          // there OR if a machine does — infrastructure selection for a module
          // declaring `zone: secure-mgmt` reads it either way (#436).
          config.managementZone === 'secure-mgmt' || (config.secureMgmtMachines ?? []).length > 0
            ? `network.secure-mgmt.subnet=${ZONE_SUBNETS['secure-mgmt']} network.secure-mgmt.gateway=${ZONE_GATEWAYS['secure-mgmt']} `
            : ''
        }dns.primary=203.0.113.1`,
    );
    if (initResult.exitCode !== 0) {
      throw new Error(`Celilo init failed: ${initResult.stderr}`);
    }
    assertFleetNameserversAreSimRoutable(projectName, composeDir);

    // Start the event-bus dispatcher (ISS-0035 / ISS-0042), now that `system
    // init` has created the bus DB. Deploys emit bus events —
    // public_web.routes_changed (caddy route reconcile), system.created (DNS
    // providers) — that only take effect when a dispatcher delivers them.
    // Production runs this as a systemd unit; the e2e mgmt box has no systemd,
    // so run it detached (nohup + disown, like the cele2e responder). One
    // dispatcher per management container serves every deploy in the test, so
    // individual tests don't need to start their own (delivery claims are
    // atomic, so a test that still does won't double-deliver).
    console.log('[progress:start] starting event dispatcher | dispatcher running');
    const dispatcherResult = dockerExec(
      projectName,
      composeDir,
      'management',
      'nohup celilo events run --poll-ms 500 --concurrency 4 > /tmp/cele2e-dispatcher.log 2>&1 < /dev/null & disown',
    );
    if (dispatcherResult.exitCode !== 0) {
      throw new Error(`Failed to start event dispatcher: ${dispatcherResult.stderr}`);
    }
  }

  console.log('[progress:done] network ready');
  // Emit project name so the runner can persist it if --keep is set
  console.log(`[e2e:project] ${projectName}`);

  return buildNetworkHandle(projectName, composeDir, celiloRoot, config.topology);
}

/**
 * Reconnect to an existing test network (previously kept alive with --keep).
 * Returns a NetworkHandle pointing to the existing Docker project. Skips
 * all the setup (image build, container start, celilo init) — assumes the
 * network is already running and healthy.
 *
 * If the project doesn't exist, throws with an actionable error.
 */
/**
 * Which topology a already-running stack was built with, read back from its
 * generated compose file. `direct-internet` is the one where fw-main holds a leg
 * on `isp-external`; everything else is `default`.
 */
function topologyFromComposeFile(composePath: string): TopologyPreset {
  try {
    const compose = parseYaml(readFileSync(composePath, 'utf-8')) as {
      services?: Record<string, { networks?: Record<string, unknown> }>;
    };
    return compose.services?.['fw-main']?.networks?.['isp-external']
      ? 'direct-internet'
      : 'default';
  } catch {
    // An unreadable compose file means the stack was not built by this
    // generator; `default` is the topology the fleet runs and the safer guess.
    return 'default';
  }
}

/**
 * Whether the generated compose wires fw-main to the secure-mgmt control-plane
 * network. The generator adds that leg as a post-step after building the
 * topology services, so it is invisible to `firewallZoneLegs(topology)` — which
 * is how a wired-but-undeclared eth4 reached the iptables converge and got
 * refused (ce-fvxd). Exported for the leg-coherence test.
 */
export function fwMainHasSecureMgmtLeg(composePath: string): boolean {
  try {
    const compose = parseYaml(readFileSync(composePath, 'utf-8')) as {
      services?: Record<string, { networks?: Record<string, unknown> }>;
    };
    return Boolean(compose.services?.['fw-main']?.networks?.['secure-mgmt']);
  } catch {
    // An unreadable compose file means the stack was not built by this
    // generator. Reporting "no leg" reproduces the under-declaration, but the
    // converge refuses loudly on the unaccounted interface, so the failure is
    // not silent — the same trade `topologyFromComposeFile` makes.
    return false;
  }
}

export function reconnectNetwork(projectName: string): NetworkHandle {
  const composeDir = PACKAGE_ROOT;

  // Verify the project actually exists
  try {
    const result = run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} ps -q`, {
      cwd: composeDir,
      timeout: 10_000,
    });
    if (!result.trim()) {
      throw new Error(`No containers found for project ${projectName}`);
    }
  } catch (err) {
    throw new Error(
      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\nFix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
    );
  }

  activeProjects.add(projectName);
  console.log(`[progress:done] reconnected to existing network (${projectName})`);
  console.log(`[e2e:project] ${projectName}`);

  // `--reuse` has no NetworkConfig to read, so the topology is recovered from the
  // compose file that built the running stack — the same artifact the generator
  // wrote. Guessing a default here would silently mis-declare the firewall's legs
  // on exactly the runs used for debugging.
  return buildNetworkHandle(
    projectName,
    composeDir,
    findCeliloRoot(),
    topologyFromComposeFile(join(composeDir, COMPOSE_FILE)),
  );
}