import { describe, expect, test } from 'bun:test'; import { readFileSync } from 'node:fs'; import { hostname } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { type DockerReader, LiveStackError, type ProcessProbe, SHARED_ORPHAN_MIN_AGE_MS, UNIT_ONLY_ENV_ENTRY, environMarksUnitOnly, findLiveE2eStack, isUnitOnlyProcess, looksLikeE2eRunCommand, } from './live-stack'; import type { LockStatus } from './run-lock'; import { startupCleanup } from './shared-infra'; /** * The live-stack guard for the startup cleanup (celilo#1297, ce-h04y). * * The #1297 incident: a second cele2e invocation got past the host-global run * lock and nukeE2eResources force-removed another run's mid-flight stack. The * guard checks Docker and the lock file at the removal site. These tests prove * the wiring through the injected runner: a live stack refuses with NOTHING * removed, a dead one sweeps. Delete the guard call and the first test goes * red — the removal commands run behind what should have been a refusal. */ const DIR = dirname(fileURLToPath(import.meta.url)); const SHARED_INFRA_SRC = readFileSync(join(DIR, 'shared-infra.ts'), 'utf-8'); /** Recording fake: answers `docker ps` from `psOutput`, records every command. */ function fakeDocker(psOutput: string): { docker: DockerReader; commands: string[] } { const commands: string[] = []; return { commands, docker: (args) => { commands.push(args.join(' ')); if (args[0] === 'ps') return psOutput; return ''; }, }; } const noLock: () => LockStatus = () => ({ free: true, holder: null, heartbeatAgeMs: null, suspect: false, ownKept: false, }); function foreignLock(overrides: Partial = {}): () => LockStatus { return () => ({ free: false, holder: { pid: 1, hostname: hostname(), session: 'polecat/ce-9999 (some other worktree)', test: 'crew-alerting', runId: 'run-abc', startedAt: new Date(Date.now() - 5 * 60_000).toISOString(), beatAt: Date.now() - 45_000, state: 'running', }, heartbeatAgeMs: 45_000, suspect: false, ownKept: false, ...overrides, }); } describe('findLiveE2eStack', () => { test('refuses when a celilo-e2e-* container is running, naming it', () => { const { docker } = fakeDocker( 'celilo-e2e-shared_namecheap-dns\trunning\ncelilo-e2e-1788769175864_fw-main\texited\n', ); // The default probe reads the REAL host process table. Under another // session's live e2e run (celilo#1332: the bun 1.4.2 grind) the // shared-only + live-runner branch fires instead of the unreadable-age // branch pinned here, and the refusal names pid counts instead of the // container. A test pinning a message shape injects its inputs: an empty // probe makes the pinned shape deterministic. const refusal = findLiveE2eStack(docker, noLock, () => []); expect(refusal).not.toBeNull(); expect(refusal?.reason).toContain('celilo-e2e-shared_namecheap-dns'); expect(refusal?.reason).not.toContain('celilo-e2e-1788769175864_fw-main'); // The remedy must be a command that EXISTS on the host printing the // message (celilo#1314): a bare `cele2e down` does not — forgejo job // workspaces are ephemeral, so the binary only exists inside a checkout. expect(refusal?.reason).toContain('docker rm -f'); expect(refusal?.reason).not.toContain('cele2e down'); // The refusal must be greppable as an environment problem, not a check // failure (celilo#1314 direction 3). expect(refusal?.reason).toContain('[infra-refusal]'); expect(refusal?.runningContainers).toEqual(['celilo-e2e-shared_namecheap-dns']); }); test('proceeds when every container is exited', () => { const { docker } = fakeDocker( 'celilo-e2e-1788769175864_fw-main\texited\ncelilo-e2e-shared_registry\tdead\n', ); expect(findLiveE2eStack(docker, noLock)).toBeNull(); }); test('treats paused and restarting containers as live — they hold real state', () => { for (const state of ['paused', 'restarting', 'running']) { const { docker } = fakeDocker(`celilo-e2e-shared_registry\t${state}\n`); const refusal = findLiveE2eStack(docker, noLock); expect(refusal?.runningContainers).toEqual(['celilo-e2e-shared_registry']); } }); test('refuses on a foreign lock with a fresh heartbeat, naming session, test and heartbeat age', () => { const { docker } = fakeDocker(''); const refusal = findLiveE2eStack(docker, foreignLock()); expect(refusal).not.toBeNull(); expect(refusal?.reason).toContain('polecat/ce-9999'); expect(refusal?.reason).toContain('crew-alerting'); expect(refusal?.reason).toContain('heartbeat 45s old'); expect(refusal?.holder?.runId).toBe('run-abc'); }); test('exempts our own lock — the caller holds it for its whole run', () => { const { docker } = fakeDocker(''); const own: () => LockStatus = () => ({ free: false, holder: { pid: process.pid, hostname: hostname(), session: 'whatever (this worktree)', test: 'caddy-internal-private', runId: 'run-own', startedAt: new Date().toISOString(), beatAt: Date.now(), state: 'running', }, heartbeatAgeMs: 0, suspect: false, ownKept: false, }); expect(findLiveE2eStack(docker, own)).toBeNull(); }); test('the container check has no own-lock exemption — a stolen lock says "ours" while another run owns the containers', () => { // The #1297 suspected path: the second invocation ends up holding the lock // while the first run's containers are still up. Only Docker can tell that // truth, so running containers refuse even when the lock is ours. const { docker } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n'); const refusal = findLiveE2eStack(docker, foreignLock({ holder: null })); expect(refusal?.runningContainers.length).toBe(1); }); }); describe('unit-only exclusion (celilo#1320)', () => { const environOf = (byPid: Record) => (pid: number): string | null => byPid[pid] ?? null; test('an environ carrying the marker is unit-only', () => { expect( environMarksUnitOnly(['PATH=/usr/bin', UNIT_ONLY_ENV_ENTRY, 'HOME=/root'].join('\0')), ).toBe(true); }); test('the marker must be a whole entry, not a substring', () => { expect(environMarksUnitOnly(['PATH=/x', 'SOMETHING_CELILO_UNIT_ONLY=12'].join('\0'))).toBe( false, ); expect(environMarksUnitOnly('CELILO_UNIT_ONLY=12')).toBe(false); }); test('an unreadable environ is NOT unit-only (inconclusive never reaps)', () => { expect(environMarksUnitOnly(null)).toBe(false); }); test('isUnitOnlyProcess reads the environ of the matched pid', () => { const environ = environOf({ 101: ['PATH=/x', UNIT_ONLY_ENV_ENTRY].join('\0'), 102: 'PATH=/x', 103: null, }); expect(isUnitOnlyProcess(101, environ)).toBe(true); expect(isUnitOnlyProcess(102, environ)).toBe(false); expect(isUnitOnlyProcess(103, environ)).toBe(false); }); test('the marker matches what test:unit actually exports', () => { const rootPackageJson = JSON.parse( readFileSync(join(DIR, '../../../package.json'), 'utf-8'), ) as { scripts: Record }; expect(rootPackageJson.scripts['test:unit']).toContain('CELILO_UNIT_ONLY=1'); }); }); describe('looksLikeE2eRunCommand', () => { test('matches the cele2e CLI by any argv mention', () => { expect(looksLikeE2eRunCommand('cele2e run smoke')).toBe(true); expect(looksLikeE2eRunCommand('./node_modules/.bin/cele2e run smoke')).toBe(true); expect(looksLikeE2eRunCommand('bun run packages/e2e/bin/cele2e.ts run --all')).toBe(true); }); test('matches a direct bun test of this package, which has no cele2e in argv', () => { expect(looksLikeE2eRunCommand('bun test packages/e2e/tests/dns-replication.test.ts')).toBe( true, ); expect(looksLikeE2eRunCommand('bun test e2e/tests/smoke.test.ts')).toBe(true); }); test('does not match unrelated bun test runs, editors, or scripts', () => { expect(looksLikeE2eRunCommand('bun test packages/mcp-server/src/tools.test.ts')).toBe(false); expect(looksLikeE2eRunCommand('vi packages/e2e/src/live-stack.test.ts')).toBe(false); expect(looksLikeE2eRunCommand('bun run packages/e2e/scripts/pack-celilo-packages.ts')).toBe( false, ); expect(looksLikeE2eRunCommand('bun build apps/celilo/src/index.ts')).toBe(false); }); }); describe('shared-only orphan reap (celilo#1314)', () => { // Docker's `{{.CreatedAt}}` format, e.g. "2026-09-07 18:49:14 +0000 UTC". const dockerAge = (msAgo: number): string => { const d = new Date(Date.now() - msAgo); return `${d.toISOString().slice(0, 10)} ${d.toISOString().slice(11, 19)} +0000 UTC`; }; const noProcesses: ProcessProbe = () => []; test('REAPS the exact shape measured on the builder: shared-only stack, 11h old, no per-test project, no cele2e process', () => { const ps = [ `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`, `celilo-e2e-shared_registry-1\trunning\t${dockerAge(11 * 3_600_000)}`, ].join('\n'); const { docker, commands } = fakeDocker(ps); // No containers at all is NOT the test: the guard must see the live shared // containers and STILL clear the way, because every piece of run evidence // is absent. expect(findLiveE2eStack(docker, noLock, noProcesses)).toBeNull(); startupCleanup('/tmp/e2e', docker, noLock, noProcesses); // The reap is a real teardown, not a refusal: the shared compose down ran. expect(commands.some((c) => c.includes(' down '))).toBe(true); }); test('must NOT reap a shared stack with a live per-test project beside it (the celilo#1297 shape)', () => { const ps = [ `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`, `celilo-e2e-1788769175864_fw-main\trunning\t${dockerAge(5 * 60_000)}`, ].join('\n'); const { docker, commands } = fakeDocker(ps); const refusal = findLiveE2eStack(docker, noLock, noProcesses); expect(refusal).not.toBeNull(); expect(refusal?.runningContainers).toContain('celilo-e2e-shared_namecheap-dns-1'); expect(refusal?.runningContainers).toContain('celilo-e2e-1788769175864_fw-main'); expect(() => startupCleanup('/tmp/e2e', docker, noLock, noProcesses)).toThrow(LiveStackError); expect(commands.some((c) => c.includes(' down '))).toBe(false); }); test('must NOT reap while any other cele2e process is alive — a run may be between suites', () => { const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`; const { docker } = fakeDocker(ps); const oneRunner: ProcessProbe = () => [424242]; const refusal = findLiveE2eStack(docker, noLock, oneRunner); expect(refusal).not.toBeNull(); expect(refusal?.reason).toContain('424242'); }); test(`must NOT reap a stack younger than ${SHARED_ORPHAN_MIN_AGE_MS / 60_000}m — the evidence is not old enough to be conclusive`, () => { const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(5 * 60_000)}`; const { docker } = fakeDocker(ps); const refusal = findLiveE2eStack(docker, noLock, noProcesses); expect(refusal).not.toBeNull(); expect(refusal?.reason).toContain('5m'); }); test('must NOT reap when the age cannot be read — inconclusive evidence refuses, it never reaps', () => { // Two fields, no CreatedAt: the pre-#1314 fake shape, and anything docker // might print that the parser does not recognize. const { docker } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n'); const refusal = findLiveE2eStack(docker, noLock, noProcesses); expect(refusal).not.toBeNull(); expect(refusal?.reason).toContain('age'); }); test('a foreign fresh lock still refuses a shared-only orphan — the reap never overrides the lock', () => { const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`; const { docker } = fakeDocker(ps); const refusal = findLiveE2eStack(docker, foreignLock(), noProcesses); expect(refusal).not.toBeNull(); expect(refusal?.reason).toContain('polecat/ce-9999'); }); test('exited containers of the shared project alone still proceed (unchanged)', () => { const ps = `celilo-e2e-shared_registry-1\texited\t${dockerAge(11 * 3_600_000)}`; const { docker } = fakeDocker(ps); expect(findLiveE2eStack(docker, noLock, noProcesses)).toBeNull(); }); }); describe('startupCleanup', () => { test('refuses behind a live stack and removes NOTHING (celilo#1297)', () => { const { docker, commands } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n'); expect(() => startupCleanup('/tmp/e2e', docker, noLock)).toThrow(LiveStackError); // Only the ps read happened. No compose down, no rm, no prune. expect(commands).toEqual([expect.stringContaining('ps -a --filter name=celilo-e2e')]); expect(commands.some((c) => c.includes(' down '))).toBe(false); expect(commands.some((c) => c.includes('rm -f'))).toBe(false); expect(commands.some((c) => c.includes('prune'))).toBe(false); }); test('sweeps when the environment is provably dead, compose down before the force sweep', () => { const { docker, commands } = fakeDocker('celilo-e2e-1788769175864_fw-main\texited\n'); startupCleanup('/tmp/e2e', docker, noLock); const down = commands.findIndex((c) => c.includes('compose -f docker-compose.shared.yml')); const rm = commands.findIndex((c) => c.includes('rm -f')); const netPrune = commands.findIndex((c) => c.includes('network prune')); const volPrune = commands.findIndex((c) => c.includes('volume prune')); expect(down).toBeGreaterThanOrEqual(0); expect(rm).toBeGreaterThan(down); expect(netPrune).toBeGreaterThan(rm); expect(volPrune).toBeGreaterThan(netPrune); // The sweep keys off the celilo-e2e prefix; it must never name the shared // project in a force removal (the graceful compose down above owns that). for (const command of commands.filter((c) => c.includes('rm -f'))) { expect(command).not.toContain('celilo-e2e-shared'); } }); }); // The guard is only real if it is WIRED — a perfect guard nobody calls is the // same failure as no guard. These source assertions pin the two removal sites // in ensureSharedInfra; the behavioral tests above pin the guard itself. describe('wiring in shared-infra.ts', () => { test('the start-of-run nuke runs through startupCleanup, which guards first', () => { expect(SHARED_INFRA_SRC).toContain('startupCleanup(e2eDir)'); const cleanupBody = SHARED_INFRA_SRC.slice( SHARED_INFRA_SRC.indexOf('export function startupCleanup'), SHARED_INFRA_SRC.indexOf('nukeE2eResources(e2eDir, docker)'), ); expect(cleanupBody).toContain('findLiveE2eStack(docker, lock, processes)'); }); test('the DNS-restart branch guards before tearing down the running stack', () => { const branchBody = SHARED_INFRA_SRC.slice( SHARED_INFRA_SRC.indexOf('DNS check failed'), SHARED_INFRA_SRC.indexOf('await stopSharedInfra()'), ); expect(branchBody).toContain('refuseOnLiveStack()'); }); });