/** * Recurrence gate for #508: a failed e2e run must not leak its Docker networks. * * The sim's networks use the same subnets as a real celilo fleet, so on a host * that lives in those subnets (celilo's forgejo-builder does) a stranded bridge * leaves a duplicate route for a production prefix and blackholes later * containers' traffic to the real address. Three consecutive release runs died * that way — in `git fetch`, ~132s connect timeout — well after the e2e run * that stranded the network had finished. * * The leak's mechanism was a wiring bug, so the gate is on the wiring. */ import { expect, test } from 'bun:test'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { networkContainersInspectCommand, projectTeardownCommands } from './container-manager'; const SRC = readFileSync( join(dirname(fileURLToPath(import.meta.url)), 'container-manager.ts'), 'utf-8', ); test("teardown is wired to 'exit', which is the event an explicit process.exit() fires", () => { // `beforeExit` alone was the bug: the runner ends with // `process.exit(failed > 0 ? 1 : 0)`, so on the failure path — where the test // never reaches handle.stop() — nothing ever tore the network down. expect(SRC).toContain("process.on('exit'"); }); test('cleanup honors --keep / --reuse so a failed run stays debuggable', () => { // Without this guard, adding the `exit` hook would destroy exactly the stack // an operator asked to keep, since --keep matters most after a failure. const body = SRC.slice( SRC.indexOf('function cleanupOnExit'), SRC.indexOf("process.on('SIGTERM'"), ); expect(body).toContain('CELILO_E2E_KEEP'); expect(body).toContain('CELILO_E2E_REUSE'); }); // The first version of this fix wired the handler correctly and still leaked, // because the teardown it ran was `docker compose -p down` with no // `-f` and no cwd. Compose resolves the project from a compose file in the // CURRENT DIRECTORY; an exit handler has no reliable cwd, so it exited "no // configuration file provided", the surrounding catch swallowed it, and nothing // was removed. Asserting the hook exists was not enough — assert it can work. test('teardown does not depend on a compose file being findable', () => { const cmds = projectTeardownCommands('celilo-e2e-1785546029645'); for (const cmd of Object.values(cmds)) { expect(cmd).not.toContain('docker compose'); expect(cmd).toContain('celilo-e2e-1785546029645'); } }); // Containers and networks were removed; volumes were not. The only thing that // would have removed them was `docker compose down --volumes`, which is the // same best-effort call that silently does nothing without a resolvable compose // file — so in practice they were never removed at all. 58 leaked // `celilo-e2e-_ssh-keys` volumes were found on the builder, the oldest // months old. A leaked volume breaks nothing on its own, which is why it went // unnoticed for months; the pile is the evidence teardown was not running. test('teardown removes volumes, not just containers and networks', () => { const cmds = projectTeardownCommands('celilo-e2e-1785546029645'); expect(cmds.listVolumes).toContain('docker volume ls'); expect(cmds.listVolumes).toContain('celilo-e2e-1785546029645'); expect(cmds.listVolumes).not.toContain('docker compose'); }); // A sim-created LXC guest is named celilo-e2e-lxc-, which neither the // per-test name filters (`celilo-e2e-1` in the startNetwork pre-sweep nor // `celilo-e2e-` in forceRemoveProject) can match. A guest left running // holds its zone network's endpoint: compose down fails that network with // "has active endpoints", the error is swallowed as best-effort, and the // subnet stays allocated. Every later suite in the run then dies in ~2s // creating that network — 44 of 54 suites in one --all run (celilo#1247). test('project teardown lists sim-created guests by label, not only by name (celilo#1247)', () => { const cmds = projectTeardownCommands('celilo-e2e-1788449289917'); expect(cmds.listGuests).toContain('docker ps -aq --filter label=celilo-e2e.project='); expect(cmds.listGuests).toContain('celilo-e2e-1788449289917'); expect(cmds.listGuests).not.toContain('docker compose'); }); test('forceRemoveProject unions the guest listing into its container sweep (celilo#1247)', () => { // The name-only sweep is what let the guest survive: the network rm that // follows ran against an endpoint that was still attached. The union must // come BEFORE the network removal in the same function. const body = SRC.slice( SRC.indexOf('function forceRemoveProject'), SRC.indexOf('function cleanupOnExit'), ); expect(body).toContain('listGuests'); expect(body.indexOf('listGuests')).toBeLessThan(body.indexOf('listNetworks')); }); test('handle.stop kills sim guests BEFORE compose down (celilo#1247)', () => { // With the guest alive, compose down fails the network ("has active // endpoints") AND the ssh-keys volume (in use), both swallowed. Removing // the guest first is what makes the normal teardown path complete. const stop = SRC.slice( SRC.indexOf('async stop(): Promise'), SRC.indexOf('export async function startNetwork'), ); expect(stop).toContain('listGuests'); expect(stop.indexOf('listGuests')).toBeLessThan(stop.indexOf('docker compose -f')); }); test('startNetwork pre-sweep sweeps guests of ANY project by label (celilo#1247)', () => { // The crash-recovery backstop must not depend on a network surviving for // the project derivation to find the guest: a guest whose network is // already gone is debris all the same. The label carries no project value // here — only the provisioner ever sets it, and the run-lock guarantees no // concurrent run owns any of them. // // The sweep lives in sweepStaleTestResources() since ce-yuzi extracted it // out of startNetwork for testability; startNetwork calls it. Assert both // halves: the function carries the label filter, startNetwork invokes it. const sweep = SRC.slice( SRC.indexOf('export function sweepStaleTestResources'), SRC.indexOf('Remove stale per-test networks'), ); expect(sweep).toContain('docker ps -aq --filter label=${GUEST_PROJECT_LABEL}`'); expect(SRC.slice(SRC.indexOf('export async function startNetwork'))).toContain( 'sweepStaleTestResources()', ); }); test('teardown cannot match the shared project', () => { // Per-test projects carry a timestamp; `celilo-e2e-shared` does not, so a // name filter for one can never match the other. Shared infra must survive — // it is torn down separately, and removing it mid-suite breaks every // subsequent test. const cmds = projectTeardownCommands('celilo-e2e-1785546029645'); for (const cmd of Object.values(cmds)) { expect(cmd).not.toContain('celilo-e2e-shared'); } }); test('teardown removes a Proxmox guest blocking a network before removing the network', () => { // ce-ywix: a guest is named celilo-e2e-lxc- and carries NO project // name, so the project-scoped container sweep never removes it. A live guest // keeps `docker network rm` failing forever, and the leaked network held the // sim's subnet — every later run then died at compose up with "Pool // overlaps". The network loop must inspect each network and remove the e2e // containers attached to it BEFORE the network rm. const cmd = networkContainersInspectCommand('celilo-e2e-1788615872911_dmz'); expect(cmd).toContain('docker network inspect'); expect(cmd).toContain('celilo-e2e-1788615872911_dmz'); const teardown = SRC.slice( SRC.indexOf('function forceRemoveProject'), SRC.indexOf('function cleanupOnExit'), ); expect(teardown).toContain('networkContainersInspectCommand'); expect(teardown).toContain('CONTAINER_PREFIX'); const inspectAt = teardown.indexOf('networkContainersInspectCommand'); const rmAt = teardown.indexOf('`docker network rm ${net}`'); expect(inspectAt).toBeGreaterThan(-1); expect(rmAt).toBeGreaterThan(-1); expect(inspectAt).toBeLessThan(rmAt); }); test("the premise holds: process.exit() fires 'exit' but NOT 'beforeExit'", async () => { // The whole fix rests on this runtime behavior. If it ever changes, the // reasoning above is void and this should fail loudly rather than silently // leaving the leak unfixed. const proc = Bun.spawn([ 'bun', '-e', "process.on('beforeExit', () => console.log('BEFORE_EXIT')); process.on('exit', () => console.log('EXIT')); process.exit(0);", ]); const out = await new Response(proc.stdout).text(); await proc.exited; expect(out).toContain('EXIT'); expect(out).not.toContain('BEFORE_EXIT'); });