/** * Lane A2 of openspec/changes/e2e-suite-recovery (ce-yuzi): the best-effort * cleanup sweeps must not report success they did not achieve. * * Every sweep step that fails is recorded as a CleanupFailure and logged as a * `[cleanup:failed]` line; the sweep itself still never throws (the exit * handler and the pre-suite recovery sweep must run to completion even when * docker is gone). Reach is measured, not reasoned about: a fake DockerCli * emulates docker's own filter semantics over a planted world, the REAL * unmodified sweep runs against it, and the rm commands it issues are compared * against the set of resources the sweep was supposed to reach. */ import { describe, expect, test } from 'bun:test'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { cleanupProgress, forceRemoveProject, sweepStaleTestResources, withDockerCli, } from './container-manager'; const PROJECT = 'celilo-e2e-1788449289917'; /** A planted docker world keyed by resource kind. Guest carries its label; * `attached` names the containers docker reports on each network. */ interface DockerWorld { containers: Record; // name -> id guests: Record; // name -> guest networks: string[]; attached: Record; // network -> attached container names volumes: string[]; } /** A fake DockerCli.exec that answers docker listing commands from `world` * using docker's own filter semantics (name filter = substring match, label * filter = exact), records every `rm` it is asked to run, and throws on * `failOn` commands. Anything it cannot answer throws — a fixture that * silently answers `''` would understate reach exactly like the real sweep * skipping a directory does. */ function fakeDocker(world: DockerWorld, failOn?: (cmd: string) => boolean) { const rms: string[] = []; const exec = (cmd: string): string => { if (failOn?.(cmd)) throw new Error(`docker: ${cmd.split(' ').slice(0, 3).join(' ')} failed`); if ( cmd.startsWith('docker rm -f ') || cmd.startsWith('docker network rm ') || cmd.startsWith('docker volume rm ') ) { rms.push(cmd); return ''; } const nameFilter = (prefix: string): string | undefined => { const m = cmd.slice(prefix.length).match(/--filter "?name=([^\s"']+)/); return m && cmd.startsWith(prefix) ? m[1] : undefined; }; // Container listing (by name and by guest label) if (cmd.startsWith('docker ps -aq')) { const label = cmd.match(/--filter label=(\S+?)(?:=(\S+))?$/); if (label) { return Object.entries(world.guests) .filter(([, g]) => label[2] === undefined || g.project === label[2]) .map(([, g]) => g.id) .join('\n'); } const nf = nameFilter('docker ps'); if (nf === undefined) throw new Error(`fakeDocker: unanswerable: ${cmd}`); return Object.entries(world.containers) .filter(([name]) => name.includes(nf)) .map(([, id]) => id) .join('\n'); } if (cmd.includes('docker network ls')) { const nf = cmd.match(/--filter name=([^\s"']+)/)?.[1]; return world.networks.filter((n) => (nf === undefined ? true : n.includes(nf))).join('\n'); } if (cmd.startsWith('docker network inspect ')) { const net = cmd.match(/docker network inspect (\S+)/)?.[1] ?? ''; const attached = (world.attached[net] ?? []).map((name, i) => [ `deadbeef${i}`, { Name: name }, ]); return JSON.stringify(Object.fromEntries(attached)); } if (cmd.startsWith('docker volume ls -q')) { const nf = cmd.match(/--filter name=([^\s"']+)/)?.[1] ?? ''; return world.volumes.filter((v) => v.includes(nf)).join('\n'); } throw new Error(`fakeDocker: unanswerable: ${cmd}`); }; return { exec, rms }; } function plantedWorld(): DockerWorld { return { containers: { [`${PROJECT}_fw-main_1`]: 'cid-fw', [`${PROJECT}_caddy_1`]: 'cid-caddy', // A live non-e2e container: no filter may ever reach it. 'nginx-live': 'cid-nginx', }, guests: { 'celilo-e2e-lxc-7001': { id: 'cid-guest', project: PROJECT }, }, networks: [`${PROJECT}_zone-a`, `${PROJECT}_zone-b`], attached: { [`${PROJECT}_zone-a`]: ['celilo-e2e-lxc-7001', 'hand-attached'], [`${PROJECT}_zone-b`]: [], }, volumes: [`${PROJECT}_ssh-keys`], }; } describe('forceRemoveProject reach (measured, not reasoned)', () => { test('removes every planted container, guest, network and volume of the project', () => { const world = plantedWorld(); const { exec, rms } = fakeDocker(world); withDockerCli({ exec, spawn: undefined as never }, () => { forceRemoveProject(PROJECT); }); const removed = rms.join('\n'); // Containers and the label-listed guest (celilo#1247). expect(removed).toContain('cid-fw'); expect(removed).toContain('cid-caddy'); expect(removed).toContain('cid-guest'); // Both networks. expect(removed).toContain(`${PROJECT}_zone-a`); expect(removed).toContain(`${PROJECT}_zone-b`); // The volume. expect(removed).toContain(`${PROJECT}_ssh-keys`); // Containers attached to a project network that carry the provisioner's // CONTAINER_PREFIX containment contract (the guest does) get their own rm. expect(rms.some((c) => c.startsWith('docker rm -f') && c.includes('celilo-e2e-lxc-7001'))).toBe( true, ); }); test('never reaches a decoy: shared infra, hand-attached containers, non-e2e', () => { const world = plantedWorld(); const { exec, rms } = fakeDocker(world); withDockerCli({ exec, spawn: undefined as never }, () => { forceRemoveProject(PROJECT); }); const removed = rms.join('\n'); expect(removed).not.toContain('nginx-live'); expect(removed).not.toContain('hand-attached'); expect(removed).not.toContain('celilo-e2e-shared'); }); }); describe('forceRemoveProject failures surface (ce-yuzi)', () => { test('a docker rm that throws is reported, not swallowed', () => { const world = plantedWorld(); const { exec } = fakeDocker(world, (cmd) => cmd.startsWith('docker rm -f')); const failures = withDockerCli({ exec, spawn: undefined as never }, () => forceRemoveProject(PROJECT), ); expect(failures.length).toBeGreaterThan(0); expect(failures.some((f) => f.step.includes('docker rm -f'))).toBe(true); expect(failures.some((f) => f.detail.includes('failed'))).toBe(true); }); test('a network rm that throws is reported, and later steps still ran', () => { const world = plantedWorld(); const { exec, rms } = fakeDocker(world, (cmd) => cmd.startsWith('docker network rm')); const failures = withDockerCli({ exec, spawn: undefined as never }, () => forceRemoveProject(PROJECT), ); expect(failures.some((f) => f.step.includes('docker network rm'))).toBe(true); // Volumes come after networks: a failed network rm must not stop them. expect(rms.some((c) => c.startsWith('docker volume rm'))).toBe(true); }); test('the sweep never throws, even when every docker command fails', () => { const { exec } = fakeDocker(plantedWorld(), () => true); const failures = withDockerCli({ exec, spawn: undefined as never }, () => forceRemoveProject(PROJECT), ); expect(failures.length).toBeGreaterThan(0); }); test('a LISTING that fails is recorded, not read as an empty result', () => { // An empty answer and a broken docker look identical to a sweep that // filters; the list failure must be counted or the sweep reports success // over resources it never saw. const { exec } = fakeDocker(plantedWorld(), (cmd) => cmd.startsWith('docker ps -aq --filter name='), ); const failures = withDockerCli({ exec, spawn: undefined as never }, () => forceRemoveProject(PROJECT), ); expect(failures.some((f) => f.step.startsWith('docker ps name='))).toBe(true); }); }); describe('sweepStaleTestResources reach (measured, not reasoned)', () => { function staleWorld(): DockerWorld { return { containers: { [`${PROJECT}_fw-main_1`]: 'cid-fw', 'celilo-e2e-shared-namecheap-dns-1': 'cid-shared-dns', 'nginx-live': 'cid-nginx', }, guests: { 'celilo-e2e-lxc-7001': { id: 'cid-guest' }, }, networks: [`${PROJECT}_zone-a`, 'celilo-e2e-shared_namecheap'], attached: {}, volumes: [`${PROJECT}_ssh-keys`, 'celilo-e2e-shared_dns'], }; } test('reaches per-test containers, guests of any project, project networks and volumes', () => { const { exec, rms } = fakeDocker(staleWorld()); const failures = withDockerCli({ exec, spawn: undefined as never }, () => sweepStaleTestResources(), ); expect(failures).toEqual([]); const removed = rms.join('\n'); expect(removed).toContain('cid-fw'); expect(removed).toContain('cid-guest'); expect(removed).toContain(`${PROJECT}_zone-a`); expect(removed).toContain(`${PROJECT}_ssh-keys`); }); test('never reaches shared infra or non-e2e resources', () => { const { exec, rms } = fakeDocker(staleWorld()); withDockerCli({ exec, spawn: undefined as never }, () => sweepStaleTestResources()); const removed = rms.join('\n'); expect(removed).not.toContain('cid-shared-dns'); expect(removed).not.toContain('nginx-live'); expect(removed).not.toContain('celilo-e2e-shared'); }); test('sweep failures are returned, not swallowed', () => { const { exec } = fakeDocker(staleWorld(), (cmd) => cmd.startsWith('docker rm -f')); const failures = withDockerCli({ exec, spawn: undefined as never }, () => sweepStaleTestResources(), ); expect(failures.length).toBeGreaterThan(0); }); }); describe('cleanupProgress is honest by construction', () => { test('says complete only on zero failures', () => { expect(cleanupProgress([])).toBe('cleanup complete'); }); test('says incomplete and names the count when steps failed', () => { const line = cleanupProgress([ { step: 'docker rm -f x', detail: 'boom' }, { step: 'docker network rm y', detail: 'boom' }, ]); expect(line).not.toContain('cleanup complete'); expect(line).toContain('2'); expect(line).toContain('[cleanup:failed]'); }); }); describe('the honest progress line is actually wired into startNetwork', () => { // Same source-assertion style as exit-cleanup.test.ts: behavior tests above // prove the helpers, this proves startNetwork really uses them. const SRC = readFileSync( join(dirname(fileURLToPath(import.meta.url)), 'container-manager.ts'), 'utf-8', ); test('startNetwork prints the sweep result after sweeping, via cleanupProgress', () => { const body = SRC.slice(SRC.indexOf('export async function startNetwork')); const sweepCall = body.indexOf('sweepStaleTestResources()'); const progressCall = body.indexOf('cleanupProgress(cleanupFailures)'); expect(sweepCall).toBeGreaterThan(-1); expect(progressCall).toBeGreaterThan(sweepCall); }); test('no path prints "cleanup complete" except through cleanupProgress', () => { // The only literal left in the file is inside cleanupProgress itself. const idx = SRC.indexOf("'cleanup complete'"); expect(idx).toBeGreaterThan(-1); expect(SRC.lastIndexOf('export function cleanupProgress', idx)).toBeGreaterThan(-1); }); });