/** * Recurrence gates for the cele2e preflight. * * Each of these conditions was, at least once, diagnosed as a product bug: the * mistake was silent and the failure surfaced later and somewhere unrelated. * Every test here breaks ONE condition deliberately and asserts the specific * message the operator gets — per Rule 7.6, a gate nobody has seen fail is not * a gate, so each assertion is written against the broken state, not the happy * one. */ import { afterAll, describe, expect, test } from 'bun:test'; import { execFileSync } from 'node:child_process'; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { BAKED_MANAGEMENT_IMAGES, type DockerfileBases, type DoctorProbe, EXPECTED_MANAGEMENT_CMD, checkBaseImages, checkDiskPressure, checkHostVm, checkImageFreshness, checkLeakedStacks, checkManagementImage, checkRunLock, checkStaleContainers, diagnose, explainBuildFailure, parseBaseImages, stackStartedAt, } from './doctor'; import type { LeakedStack } from './doctor'; import type { HostFacts, HostVmFacts } from './host-vm'; import type { LockHolder, LockStatus } from './run-lock'; import { PUBLISHED_FINGERPRINT_PREFIX, SOURCE_LABEL, computeSourceFingerprint, } from './source-fingerprint'; /** A probe where everything is healthy; each test breaks exactly one thing. */ function healthyProbe(overrides: Partial = {}): DoctorProbe { return { imageExists: () => true, imageCmd: () => [EXPECTED_MANAGEMENT_CMD], imageLabel: () => null, celiloVersion: () => '1.2.3', staleContainers: () => [], reclaimableImageBytes: () => 0, leakedStacks: () => [], ...overrides, }; } function holder(overrides: Partial = {}): LockHolder { return { pid: 4242, hostname: 'testhost', session: 'branch (/work/tree)', test: 'caddy-direct-internet', runId: 'r1', startedAt: new Date().toISOString(), beatAt: Date.now(), state: 'running', ...overrides, }; } function lockState(overrides: Partial = {}): LockStatus { return { free: true, holder: null, heartbeatAgeMs: null, suspect: false, ownKept: false, ...overrides, }; } describe('management image (problem 1: the bake fails as an SSH error)', () => { test('a missing baked image fails preflight naming build-infra as the fix', () => { const check = checkManagementImage( healthyProbe({ imageExists: (ref) => ref !== 'celilo-e2e/management:latest' }), ); expect(check.status).toBe('fail'); expect(check.detail).toContain('celilo-e2e/management:latest'); expect(check.detail).toContain('cannot build'); expect(check.remedy).toBe('cele2e build-infra'); }); test('a bake that lost its Cmd fails, and says so as the SSH error it would become', () => { // The bake container runs `sleep infinity`; docker commit persists that Cmd // unless overridden. Without /startup.sh the ssh-keys volume is never // populated and `machine add` dies with a connection error against a // firewall IP — which reads as a network bug and is not one. const check = checkManagementImage(healthyProbe({ imageCmd: () => ['sleep', 'infinity'] })); expect(check.status).toBe('fail'); expect(check.detail).toContain(EXPECTED_MANAGEMENT_CMD); expect(check.detail).toContain('ssh-keys'); expect(check.detail).toContain('Cannot connect to root@'); expect(check.remedy).toBe('cele2e build-infra'); }); test('a hollow image — starts fine, no celilo inside — FAILS', () => { // celilo#1390: doctor said "2 baked images present, both running // /startup.sh" about an image with no celilo binary at all. const check = checkManagementImage(healthyProbe({ celiloVersion: () => null })); expect(check.status).toBe('fail'); expect(check.detail).toContain('HOLLOW'); expect(check.detail).toContain('127'); expect(check.remedy).toBe('cele2e build-infra'); }); test('a passing check reports the version it found, so a stale bake is visible too', () => { const check = checkManagementImage(healthyProbe({ celiloVersion: () => '0.42.1' })); expect(check.status).toBe('ok'); expect(check.detail).toContain('0.42.1'); }); test('both images present and correctly baked passes', () => { const check = checkManagementImage(healthyProbe()); expect(check.status).toBe('ok'); expect(check.detail).toContain(String(BAKED_MANAGEMENT_IMAGES.length)); }); }); describe('base images (problem 3: a missing image reads as a network failure)', () => { const dockerfiles: DockerfileBases[] = [ { derived: 'celilo-e2e/observer', bases: ['ubuntu:22.04'] }, { derived: 'celilo-e2e/pebble', bases: ['ghcr.io/letsencrypt/pebble:latest'] }, ]; test('a base image missing with its derived image ALSO missing fails, with the pull', () => { // Post `docker image prune`: nothing local, so the build must go to a // registry — the exact state that produced "failed to solve: ubuntu:22.04: // net/http: TLS handshake timeout" 16 images into a rebuild. const check = checkBaseImages(healthyProbe({ imageExists: () => false }), dockerfiles); expect(check.status).toBe('fail'); expect(check.detail).toContain('ubuntu:22.04'); expect(check.remedy).toContain('docker pull ubuntu:22.04'); expect(check.remedy).toContain('docker pull ghcr.io/letsencrypt/pebble:latest'); }); test('a base image missing but its derived image present passes — the build is cached', () => { // This is the ordinary steady state on a working machine, and a check that // failed here would refuse environments that run fine today. const check = checkBaseImages( healthyProbe({ imageExists: (ref) => ref.startsWith('celilo-e2e/') }), dockerfiles, ); expect(check.status).toBe('ok'); }); test('only the images that actually need fetching are named', () => { const check = checkBaseImages( healthyProbe({ imageExists: (ref) => ref !== 'ubuntu:22.04' && ref !== 'celilo-e2e/observer', }), dockerfiles, ); expect(check.status).toBe('fail'); expect(check.detail).toContain('ubuntu:22.04'); expect(check.detail).not.toContain('pebble'); }); }); describe('run-lock (problems 2 and 5)', () => { test('a stale heartbeat with a live PID is SUSPECT, not merely busy', () => { // The one hang PID-liveness cannot see: build-infra wedged at step [16/27], // process alive and sleeping, heartbeat 1913s stale. Nothing surfaced it. const check = checkRunLock( lockState({ free: false, holder: holder({ beatAt: Date.now() - 1_913_000 }), heartbeatAgeMs: 1_913_000, suspect: true, }), ); expect(check.status).toBe('fail'); expect(check.detail).toContain('SUSPECT'); expect(check.detail).toContain('31m'); expect(check.detail).toContain('not progressing'); expect(check.remedy).toContain('4242'); }); test("this session's own kept lock is a warning, not contention", () => { const check = checkRunLock( lockState({ free: false, holder: holder({ state: 'kept' }), heartbeatAgeMs: 0, ownKept: true, }), ); expect(check.status).toBe('warn'); expect(check.detail).toContain('auto-releases'); }); test("another session's kept lock still fails", () => { const check = checkRunLock( lockState({ free: false, holder: holder({ state: 'kept' }), heartbeatAgeMs: 0 }), ); expect(check.status).toBe('fail'); expect(check.detail).toContain('another session'); }); test('a healthy busy lock reports the heartbeat age', () => { const check = checkRunLock( lockState({ free: false, holder: holder(), heartbeatAgeMs: 11_000 }), ); expect(check.status).toBe('fail'); expect(check.detail).toContain('11s ago'); expect(check.detail).not.toContain('SUSPECT'); }); }); describe('stale containers and disk pressure', () => { test('leftover containers with no live run warn and point at `cele2e down`', () => { const check = checkStaleContainers( healthyProbe({ staleContainers: () => ['celilo-e2e-1_management', 'celilo-e2e-1_fw-main'] }), lockState(), ); expect(check.status).toBe('warn'); expect(check.remedy).toContain('cele2e down'); }); test('containers belonging to a live holder are not debris', () => { const check = checkStaleContainers( healthyProbe({ staleContainers: () => ['celilo-e2e-1_management'] }), lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }), ); expect(check.status).toBe('ok'); }); test('large reclaimable space offers the safe prune and names the destructive one', () => { // The two forms are not interchangeable and the difference is the whole // point: bare `docker image prune` removes UNTAGGED images (superseded // management bakes, the bulk of this pile), while `-a` and `system prune` // also remove the TAGGED base images the next build needs — which is the // reflex that costs a full 27-image rebuild and then reads as a network // failure. The remedy must say which is which. const check = checkDiskPressure(healthyProbe({ reclaimableImageBytes: () => 40 * 1024 ** 3 })); expect(check.status).toBe('warn'); expect(check.remedy).toContain('docker image prune -f'); expect(check.remedy).toContain('NEVER `-a`'); expect(check.remedy).toContain('docker system prune'); }); test('disk pressure never blocks a run', () => { const report = diagnose({ pkgDir: '/nonexistent', probe: healthyProbe({ reclaimableImageBytes: () => 99 * 1024 ** 3 }), lock: lockState(), }); expect(report.checks.find((c) => c.name === 'disk')?.status).toBe('warn'); expect(report.ok).toBe(true); }); }); describe('leaked per-test stacks (ce-ywix: a crashed suite held a subnet for 2.5h)', () => { // The mechanism: a Proxmox guest (celilo-e2e-lxc-) carries no project // name, survives every project-scoped sweep, and keeps the network attached — // so every later run dies at compose up with "Pool overlaps". The check names // the leak, its suite-of-origin timestamp, and the exact removal commands. const leaked: LeakedStack[] = [ { network: 'celilo-e2e-1788615872911_dmz', containers: ['celilo-e2e-lxc-104'], }, ]; test('a leaked stack with no live run warns and names the exact removal commands', () => { const check = checkLeakedStacks(healthyProbe({ leakedStacks: () => leaked }), lockState()); expect(check.status).toBe('warn'); expect(check.detail).toContain('celilo-e2e-1788615872911_dmz'); expect(check.detail).toContain('celilo-e2e-lxc-104'); expect(check.remedy).toContain('docker rm -f celilo-e2e-lxc-104'); expect(check.remedy).toContain('docker network rm celilo-e2e-1788615872911_dmz'); }); test('the suite-of-origin is attributable from the project timestamp', () => { // The bead's attribution note: the project name embeds epoch-ms, so the // leaker is identifiable after the fact without any other record. const check = checkLeakedStacks(healthyProbe({ leakedStacks: () => leaked }), lockState()); expect(check.detail).toContain('suite started'); expect(check.detail).toMatch(/suite started \d+[smh]/); }); test('a stack blocked by a NON-e2e container fails — the sweep cannot self-heal it', () => { const check = checkLeakedStacks( healthyProbe({ leakedStacks: () => [ { network: 'celilo-e2e-1788615872911_dmz', containers: ['my-dev-box'] }, ], }), lockState(), ); expect(check.status).toBe('fail'); expect(check.detail).toContain('my-dev-box'); expect(check.remedy).toContain( 'docker network disconnect -f celilo-e2e-1788615872911_dmz my-dev-box', ); expect(check.remedy).toContain('docker network rm celilo-e2e-1788615872911_dmz'); // Never instruct removing a container the sweep does not own. expect(check.remedy).not.toContain('docker rm -f my-dev-box'); }); test("stacks belonging to a live holder are that run's, not debris", () => { const check = checkLeakedStacks( healthyProbe({ leakedStacks: () => leaked }), lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }), ); expect(check.status).toBe('ok'); }); test('the run\'s own preflight treats its held lock as "no legitimate holder"', () => { // skipLock in the runner means WE hold the lock, so at preflight time any // stack in sight is debris even though lockStatus() reads busy (our own // holder record). Attributing it to "the holder" would hide the leak from // the exact surface that exists to name it. const check = checkLeakedStacks( healthyProbe({ leakedStacks: () => leaked }), lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }), true, ); expect(check.status).toBe('warn'); }); test('no stacks reads clean', () => { expect(checkLeakedStacks(healthyProbe(), lockState()).status).toBe('ok'); }); test('the shared and interactive stacks are never mistaken for per-test debris', () => { expect(stackStartedAt('celilo-e2e-shared_real-internet')).toBeNull(); expect(stackStartedAt('celilo-e2e-interactive_dmz')).toBeNull(); expect(stackStartedAt('celilo-e2e-1788615872911_dmz')).toBe(1788615872911); }); }); describe('Dockerfile base parsing', () => { test("ignores references to the Dockerfile's own build stages", () => { const bases = parseBaseImages( [ 'FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS fetch', 'RUN apt-get update', 'FROM --platform=linux/amd64 eclipse-temurin:25-jre', 'COPY --from=fetch /out /out', ].join('\n'), ); expect(bases).toEqual(['debian:bookworm-slim', 'eclipse-temurin:25-jre']); }); test('a plain single-stage Dockerfile yields its one base', () => { expect(parseBaseImages('FROM ubuntu:22.04\nRUN echo hi\n')).toEqual(['ubuntu:22.04']); }); }); describe('build failure translation', () => { test('a "failed to solve: " is explained as a missing local image', () => { const text = explainBuildFailure( 'ERROR: failed to solve: ubuntu:22.04: failed to authorize: ... net/http: TLS handshake timeout', ); expect(text).toContain('not in the local image store'); expect(text).toContain('docker pull ubuntu:22.04'); }); test('a genuine RUN-step failure is left alone', () => { expect( explainBuildFailure( 'ERROR: failed to solve: process "/bin/sh -c apt-get update" did not complete successfully: exit code: 100', ), ).toBe(''); }); }); describe('diagnose', () => { test('any failing check makes the whole report not ok', () => { const report = diagnose({ pkgDir: '/nonexistent', probe: healthyProbe({ imageExists: () => false }), lock: lockState(), }); expect(report.ok).toBe(false); expect(report.checks.find((c) => c.name === 'management-image')?.status).toBe('fail'); }); test('skipLock omits the lock check — run holds the lock itself', () => { const report = diagnose({ pkgDir: '/nonexistent', probe: healthyProbe(), lock: lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }), skipLock: true, }); expect(report.checks.some((c) => c.name === 'run-lock')).toBe(false); expect(report.ok).toBe(true); }); }); // ─── The Docker host VM ────────────────────────────────────────────── // // Two settings, each measured on 2026-09-05 (see host-vm.ts). Warnings rather // than failures: a badly-shaped VM makes a run slow, not impossible, and this // check's job is to stop that being invisible. const M1_PRO: HostFacts = { cpus: 10, memoryGiB: 32 }; function vm(overrides: Partial = {}): HostVmFacts { return { profile: 'default', cpus: 8, memoryGiB: 12, mountType: 'virtiofs', vmType: 'vz', ...overrides, }; } describe('checkHostVm', () => { test('a VM inside the policy is ok and says what it found', () => { const check = checkHostVm(vm(), M1_PRO); expect(check.status).toBe('ok'); expect(check.detail).toContain('12 GiB'); expect(check.detail).toContain('virtiofs'); }); test('no VM at all is ok — docker runs natively on Linux', () => { const check = checkHostVm(null, M1_PRO); expect(check.status).toBe('ok'); expect(check.detail).toContain('no colima VM'); }); test('an oversized VM warns, because the host pays for the guest page cache', () => { // The measured state: 24 of 32 GiB, host swapping 21.9 GiB. const check = checkHostVm(vm({ memoryGiB: 24 }), M1_PRO); expect(check.status).toBe('warn'); expect(check.detail).toContain('24 GiB'); // Restarting fixes memory, so it must NOT demand the destructive path. expect(check.remedy).toContain('cele2e host up'); }); test('an sshfs VM warns AND routes to reset, because colima cannot change it in place', () => { const check = checkHostVm(vm({ mountType: 'sshfs' }), M1_PRO); expect(check.status).toBe('warn'); expect(check.detail).toContain('sshfs'); expect(check.remedy).toContain('cele2e host reset'); }); test('a qemu VM is not asked for virtiofs, which only Apple Virtualization has', () => { const check = checkHostVm(vm({ vmType: 'qemu', mountType: 'sshfs' }), M1_PRO); expect(check.status).toBe('ok'); }); test('more VM CPUs than the host has warns', () => { const check = checkHostVm(vm({ cpus: 16 }), M1_PRO); expect(check.status).toBe('warn'); expect(check.detail).toContain('16 CPUs'); }); }); // ─── Image freshness ───────────────────────────────────────────────── // // The baked management image carries the celilo CLI every test exercises. A // stale one fails for reasons the working tree cannot explain, and says // nothing about it — so the fingerprint is compared, never assumed. /** * A throwaway checkout, so these tests assert on a tree they control rather * than on whatever the ambient worktree happens to hold. A test whose subject * is "the repo I am running in" passes or fails for reasons unrelated to the * code under test. */ function makeCheckout(): string { const root = mkdtempSync(join(tmpdir(), 'fingerprint-')); mkdirSync(join(root, 'apps', 'celilo', 'src'), { recursive: true }); writeFileSync(join(root, 'apps', 'celilo', 'package.json'), '{"name":"@celilo/cli"}\n'); writeFileSync(join(root, 'apps', 'celilo', 'src', 'index.ts'), 'export const a = 1;\n'); const git = (...args: string[]) => execFileSync('git', ['-C', root, ...args], { stdio: 'ignore' }); git('init', '-q'); git('config', 'user.email', 'test@celilo.invalid'); git('config', 'user.name', 'test'); git('add', '-A'); git('commit', '-qm', 'initial'); return root; } const REPO_ROOT = makeCheckout(); afterAll(() => rmSync(REPO_ROOT, { recursive: true, force: true })); describe('checkImageFreshness', () => { test('no checkout means nothing to be stale against', () => { const check = checkImageFreshness(healthyProbe(), null); expect(check.status).toBe('ok'); expect(check.detail).toContain('no celilo checkout'); }); test('an unstamped image warns, because its age cannot be established', () => { // repoRoot is this checkout, so a fingerprint IS computed; the image has no label. const check = checkImageFreshness(healthyProbe({ imageLabel: () => null }), REPO_ROOT); expect(check.status).toBe('warn'); expect(check.detail).toContain('no source stamp'); expect(check.remedy).toBe('cele2e build-infra'); }); test('a stamp from other source warns and names both fingerprints', () => { const check = checkImageFreshness( healthyProbe({ imageLabel: () => 'deadbeefdeadbeef' }), REPO_ROOT, ); expect(check.status).toBe('warn'); expect(check.detail).toContain('deadbeefdeadbeef'); expect(check.detail).toContain('NOT the code in this checkout'); }); test('a published bake is ok — it was never meant to match this tree', () => { const check = checkImageFreshness( healthyProbe({ imageLabel: () => `${PUBLISHED_FINGERPRINT_PREFIX}celilo 2.1.0` }), REPO_ROOT, ); expect(check.status).toBe('ok'); expect(check.detail).toContain('real npm'); }); test('a matching stamp is ok', () => { const expected = computeSourceFingerprint(REPO_ROOT); expect(expected).not.toBeNull(); const check = checkImageFreshness( healthyProbe({ imageLabel: () => expected as string }), REPO_ROOT, ); expect(check.status).toBe('ok'); expect(check.detail).toContain('baked from this tree'); }); test('it reads the label the bake actually writes', () => { let asked = ''; checkImageFreshness( healthyProbe({ imageLabel: (_ref, label) => { asked = label; return null; }, }), REPO_ROOT, ); expect(asked).toBe(SOURCE_LABEL); }); });