/** * Shared Infrastructure Manager * * Starts the shared infrastructure (DNS hierarchy, Pebble ACME, * registry, fw-ext) once per test suite run. Per-test containers * connect to the shared networks via Docker's external network * references. * * The shared infra project name is deterministic (celilo-e2e-shared) so * it can be referenced by per-test compose files and cleaned up reliably. */ import { execSync } from 'node:child_process'; import { copyFileSync, existsSync, rmSync, statSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { SHARED_NETWORKS, SHARED_PROJECT_NAME, generateSharedInfraYaml, prepareRegistryDropZone, } from './docker-compose-generator'; import { type DockerReader, LiveStackError, type ProcessProbe, findLiveE2eStack, realDocker, realProcessProbe, refuseOnLiveStack, } from './live-stack'; import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from './registry-bundle'; import { type LockStatus, lockStatus } from './run-lock'; const SHARED_COMPOSE_FILE = 'docker-compose.shared.yml'; const COMPOSE_TIMEOUT = 120_000; function run(cmd: string, opts?: { cwd?: string; timeout?: number }): string { return execSync(cmd, { timeout: opts?.timeout ?? COMPOSE_TIMEOUT, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'], cwd: opts?.cwd, }).trim(); } /** * Remove ALL celilo-e2e-* Docker resources by name prefix (#212). Used at * start-of-run, where the live-stack guard (findLiveE2eStack, called by * startupCleanup before this runs) has verified nothing live remains to * remove. Graceful compose-down of the shared project first (clean network * detach), then a force sweep that catches orphans from any prior invocation * regardless of compose-project name or the path that created them. * * Docker access goes through the injected runner (ce-h4no seam) so tests can * prove the guard fires BEFORE any removal command runs. */ function nukeE2eResources(e2eDir: string, docker: DockerReader): void { try { docker( [ 'compose', '-f', SHARED_COMPOSE_FILE, '-p', SHARED_PROJECT_NAME, 'down', '--volumes', '--remove-orphans', ], { cwd: e2eDir, timeoutMs: 30_000 }, ); } catch {} try { const ids = docker(['ps', '-aq', '--filter', 'name=celilo-e2e'], { timeoutMs: 10_000 }); if (ids.trim()) docker(['rm', '-f', ...ids.split('\n').filter(Boolean)], { timeoutMs: 60_000 }); } catch {} try { const nets = docker(['network', 'ls', '--format', '{{.Name}}']) .split('\n') .filter((n) => n.startsWith('celilo-e2e')); for (const net of nets) { try { docker(['network', 'rm', net], { timeoutMs: 5_000 }); } catch {} } } catch {} try { docker(['network', 'prune', '-f'], { timeoutMs: 10_000 }); } catch {} try { docker(['volume', 'prune', '-f'], { timeoutMs: 10_000 }); } catch {} } /** * The start-of-run cleanup: refuse if a live stack is in the way * (celilo#1297), otherwise sweep every celilo-e2e-* resource. The refusal * throws LiveStackError — ensureSharedInfra turns it into exit 3 at its two * call sites — so tests can drive this with a fake runner and assert the * sweep removes nothing behind a refusal. */ export function startupCleanup( e2eDir: string, docker: DockerReader = realDocker, lock: () => LockStatus = lockStatus, processes: ProcessProbe = realProcessProbe, ): void { const refusal = findLiveE2eStack(docker, lock, processes); if (refusal) throw new LiveStackError(refusal); console.log( '[progress:start] cleaning up stale shared infrastructure | shared infra cleanup complete', ); nukeE2eResources(e2eDir, docker); } /** * Check if the shared infrastructure is already running and healthy. * * "Healthy" means BOTH: * - At least one celilo-e2e-shared container is running (per * `docker compose ps -q`). * - All expected shared networks exist (a previous aggressive * `docker network prune` or `docker network rm` can remove * networks while leaving containers nominally "up" — those * containers retain the in-kernel network state but compose * can't attach NEW containers to a missing network. Confirmed * bitten 2026-05-06: shared containers were all running but * `celilo-e2e-shared_real-internet` had been pruned, and the * per-test stack failed with "network … declared as external, * but could not be found" when starting per-test services). * * Source of truth is always Docker — each test file runs in a fresh * bun process, so there is no reliable in-process cache to lean on. */ export function isSharedInfraRunning(): boolean { try { const containers = run( `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} ps -q`, { cwd: getE2eDir() }, ); if (containers.trim().length === 0) { return false; } const networks = run('docker network ls --format "{{.Name}}"'); const networkLines = networks.split('\n'); const haveAll = Object.values(SHARED_NETWORKS).every((n) => networkLines.includes(n)); if (!haveAll) { return false; } return true; } catch { return false; } } function getE2eDir(): string { return join(__dirname, '..'); } /** * Start the shared infrastructure. Idempotent — if already running, * verifies health and returns immediately. */ export async function ensureSharedInfra(): Promise { if (isSharedInfraRunning()) { // Verify DNS is still healthy try { const result = run( `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} exec -T namecheap-dns dig @127.0.0.1 iamtheinternet.org SOA +short +timeout=2`, { cwd: getE2eDir(), timeout: 10_000 }, ); if (result.trim().length > 0) { return; // Shared infra healthy, nothing to do } } catch { // DNS check failed — restart shared infra. The restart tears down the // running stack, so the live-stack guard fires first (celilo#1297): a // stack that is up is never force-removed, even when its DNS is wedged. // The operator clears it with the docker removal named in the refusal // (a bare `cele2e down` does not exist on ephemeral CI hosts — // celilo#1314) and re-runs. This retires // the automatic DNS-restart self-heal by decision (peba, 2026-09-07): // fail loudly naming the holder beats silently clobbering a stack that // might be another session's — the #1297 incident rode exactly this // branch. console.log( '[progress:start] shared infra DNS check failed, restarting | shared infra restarted', ); refuseOnLiveStack(); await stopSharedInfra(); } } const e2eDir = getE2eDir(); // Start-of-run cleanup, behind the live-stack guard. The run-lock held here // (acquired in the runner / build path before any docker mutation) is no // longer treated as proof that no other session's stack is live — the #1297 // incident got past it. startupCleanup checks Docker and the lock file at // the removal site and refuses with exit 3 when anything live remains. try { startupCleanup(e2eDir); } catch (err) { if (err instanceof LiveStackError) { console.error(`\n${err.message}\n`); process.exit(3); } throw err; } // Refresh the bundled registry-server source so Dockerfile.registry's // COPY resolves whether we're in the monorepo (regenerated from the // sibling) or installed from npm (already bundled in the tarball). ensureRegistryServerBundle(e2eDir); // Same reason, for Dockerfile.proxmox-sim: the simulator runs the fake from // this checkout rather than whatever npm last published. ensureTerraformFakeBundle(e2eDir); // Generate and write compose file const yaml = generateSharedInfraYaml(); writeFileSync(join(e2eDir, SHARED_COMPOSE_FILE), yaml); // The registry's /uploads bind source must exist before `up` (a missing // bind source becomes an empty root-owned dir), and in the monorepo it // starts empty so no stale .netapp shadows live module source. prepareRegistryDropZone(); // Seed the live DNS zone files from their templates BEFORE the compose // mounts them. config/dns/{iamtheinternet.org,example.net}.zone are // gitignored runtime state (scrubDnsZones rewrites them per-test); on a // fresh checkout they don't exist yet, and a Docker bind-mount of a missing // host path silently creates an empty DIRECTORY — so Knot serves no zone and // DNS convergence times out. Seeding here makes the bind-mount targets real // files, so shared infra bootstraps from a clean checkout (e.g. CI builder). for (const zone of ['iamtheinternet.org', 'example.net']) { const tpl = join(e2eDir, 'config', 'dns', 'templates', `${zone}.zone`); const live = join(e2eDir, 'config', 'dns', `${zone}.zone`); // Remove an empty dir left by a prior bind-mount on a missing path. if (existsSync(live) && statSync(live).isDirectory()) { rmSync(live, { recursive: true, force: true }); } copyFileSync(tpl, live); } // Start. The generated compose carries `image:` only (never `build:`), so // `up` cannot build and cannot resolve a FROM from docker.io; a missing // baked tag fails right here, naming the image. startNetwork checks the // full referenced set first and names the remedy (cele2e build-infra). console.log('[progress:start] starting shared infrastructure | shared infra started'); run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} up -d`, { cwd: e2eDir, timeout: 120_000, }); // Wait for DNS convergence — robustly. The old single 60s window with no // recovery was an intermittent build-infra failure on the builder (#319: // namecheap-dns hadn't served its zone within 60s, and the bare throw gave // no clue why). Now: a longer window, RESTART the DNS container if it's // stuck (re-triggers the zone load), and DUMP diagnostics before giving up. console.log('[progress:start] waiting for DNS convergence | DNS converged'); const dnsTimeout = 180_000; const start = Date.now(); let lastRestart = start; while (Date.now() - start < dnsTimeout) { try { const result = run( `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} exec -T namecheap-dns dig @127.0.0.1 iamtheinternet.org SOA +short +timeout=2`, { cwd: e2eDir, timeout: 10_000 }, ); if (result.trim().length > 0) { console.log('[progress:done] shared infrastructure ready'); return; } } catch { // retry } // Recovery: if ~45s have passed since the last (re)start without // converging, the DNS container likely didn't load its zone — restart // it to re-trigger the load instead of waiting out the whole window. if (Date.now() - lastRestart > 45_000) { console.log('[progress:sub] DNS not converged yet — restarting namecheap-dns'); try { run( `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} restart namecheap-dns`, { cwd: e2eDir, timeout: 30_000 }, ); } catch {} lastRestart = Date.now(); } // e2e-sleep-ok: poll cadence; convergence is re-checked each iteration, diagnostics on timeout. await new Promise((r) => setTimeout(r, 2000)); } // Diagnostics before failing, so the log shows WHY it didn't converge. let psOut = ''; try { psOut = run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} ps`, { cwd: e2eDir, timeout: 10_000, }); } catch {} let dnsLog = ''; try { dnsLog = run( `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} logs --tail 40 namecheap-dns`, { cwd: e2eDir, timeout: 10_000 }, ); } catch {} throw new Error( `Timeout waiting for shared infrastructure DNS convergence (${dnsTimeout / 1000}s)\n` + `--- compose ps ---\n${psOut}\n--- namecheap-dns logs (tail 40) ---\n${dnsLog}`, ); } /** * Stop the shared infrastructure. Called once at the end of the test suite. */ export async function stopSharedInfra(): Promise { console.log('[progress:start] stopping shared infrastructure | shared infra stopped'); const e2eDir = getE2eDir(); try { run( `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} down --volumes --remove-orphans`, { cwd: e2eDir, timeout: 60_000 }, ); } catch { // Best effort } }