/** * Live-stack guard for the e2e startup cleanup (celilo#1297, ce-h04y). * * nukeE2eResources tears down EVERY celilo-e2e-* container by name prefix, on * the assumption that holding the run-lock proves no other session's stack is * live. The #1297 incident broke that assumption: a second invocation got past * the lock anyway and force-removed another run's mid-flight stack, nine * containers, the suite dead at 12.95s with exit 137. Which lock path let it * through is not reconstructable, and does not need to be — the guard defends * every path by checking ground truth (Docker and the lock file) at the * removal site instead of trusting the lock's own verdict. * * Decided (peba, 2026-09-07, option a): before the startup cleanup removes * anything, it refuses when any celilo-e2e-* container is still live or the * run-lock heartbeat is fresh and foreign. It removes only what is provably * dead. There is no --force: an operator who wants a live stack gone uses * `cele2e down`. * * Deliberately NOT guarded: the runner's end-of-run stopSharedInfra teardown. * The owner tearing down its own live stack is the normal exit path, and the * guard's container check would refuse it by definition. * * The shared-stack orphan reap (celilo#1314). The #1297 guard is keyed on * "any live celilo-e2e-* container", and the shared stack is designed never to * be touched by name-protecting cleanup — so one abnormal exit wedges the * host permanently: the orphaned shared stack refuses every later cleanup, it * is the one thing the guard structurally cannot resolve. Measured on the * builder 2026-09-08: 13 celilo-e2e-shared-* containers, 11h old, no per-test * project, no cele2e process, and every npm-consumer-smoke run red on a check * that never executed. So a stack whose ONLY live containers belong to the * shared project gets an evidence test instead of a blanket refusal: no * foreign lock, no live run process, and an age past a threshold means the * stack is garbage and cleanup may act. Any single piece of run evidence * present, or any age that cannot be read, refuses — inconclusive evidence * never reaps, because a false reap kills a real run while a false refusal * costs the operator one docker command. * * The refusal itself names a command that exists on the host printing it (a * bare `cele2e down` does not: forgejo job workspaces are ephemeral, so the * binary only exists inside a checkout), and every refusal reason carries the * `[infra-refusal]` marker with its exit code, so a log search distinguishes * an environment problem from a check failure (celilo#1314 directions 2+3). * * The unit-only exclusion (celilo#1320). ci/validate runs `bun run test:unit` * on the same persistent builder as npm-consumer-smoke, and every `bun test` * child it spawns matched the probe's bun-test arm while managing no Docker * at all — the smoke job's teardown then declined to clear a finished run's * shared stack, and the next run inside the orphan hour refused. A process * whose environ carries CELILO_UNIT_ONLY=1 (the root script exports it; every * child inherits) is a unit-test run and is not run evidence. An unreadable * environ keeps the match: the exclusion follows the same bias as the * evidence test itself, inconclusive never reaps. */ import type { ExecFileSyncOptions } from 'node:child_process'; import { execFileSync } from 'node:child_process'; import { readFileSync } from 'node:fs'; import { SHARED_PROJECT_NAME } from './docker-compose-generator'; import { type LockHolder, type LockStatus, formatAge, formatBusy, heartbeatAgeMs, isSameSession, lockStatus, } from './run-lock'; /** * Docker access, injected so the guard and the sweep it protects are * unit-testable without a daemon (the seam lane from ce-h4no). Unlike * proxmox-provisioner's DockerRunner this carries per-call timeouts and cwd, * which the compose sweep needs. */ export type DockerReader = (args: string[], opts?: { timeoutMs?: number; cwd?: string }) => string; export const realDocker: DockerReader = (args, opts) => execFileSync('docker', args, { encoding: 'utf-8', timeout: opts?.timeoutMs ?? 60_000, cwd: opts?.cwd, stdio: ['pipe', 'pipe', 'pipe'], } satisfies ExecFileSyncOptions).trim(); /** What the guard found, and the human-readable refusal naming it. */ export interface LiveStackRefusal { reason: string; /** Container names still live, when the refusal is about containers. */ runningContainers: string[]; /** The foreign lock holder, when the refusal is about the lock. */ holder: LockHolder | null; } export class LiveStackError extends Error { constructor(readonly refusal: LiveStackRefusal) { super(refusal.reason); this.name = 'LiveStackError'; } } /** * How old the youngest live shared-stack container must be before a * shared-only stack with no other run evidence is treated as an orphan. * One hour sits far above any window in which a run process could have died * without its containers dying too, and far below the 11h the builder sat * wedged (celilo#1314). A stack younger than this refuses even with no other * evidence: the reap is the dangerous direction, so it waits for certainty. */ export const SHARED_ORPHAN_MIN_AGE_MS = 60 * 60_000; /** * Live pids of processes that may own an e2e run. Injectable so the reap's * evidence test is unit-testable without a process table. */ export type ProcessProbe = () => number[]; /** * The env entry `bun run test:unit` exports (root package.json). A process * carrying it is a unit-test run: the e2e unit suites exercise Docker only * through injected readers, so it never manages the shared stack. */ export const UNIT_ONLY_ENV_ENTRY = 'CELILO_UNIT_ONLY=1'; /** * Raw environment of one pid (NUL-separated entries), or null when it cannot * be read — /proc is Linux-only, and a foreign-uid environ is unreadable even * where /proc exists. */ export type EnvironReader = (pid: number) => string | null; export const realEnvironReader: EnvironReader = (pid) => { try { return readFileSync(`/proc/${pid}/environ`, 'utf8'); } catch { return null; } }; /** * Does a raw environ block mark its process as unit-only? Pure so the * probe's exclusion is testable without a filesystem. A null (unreadable) * environ is NOT unit-only: an unreadable answer is a missing answer, and the * probe biases to refusing — a false refusal costs the operator one docker * command, a false miss reaps a live run. */ export function environMarksUnitOnly(environ: string | null): boolean { if (environ === null) return false; return environ.split('\0').includes(UNIT_ONLY_ENV_ENTRY); } /** The exclusion realProcessProbe applies to its matches; injectable for tests. */ export function isUnitOnlyProcess( pid: number, environ: EnvironReader = realEnvironReader, ): boolean { return environMarksUnitOnly(environ(pid)); } /** * Does this process command line look like an e2e run? Pure so the probe's * reach is testable. Matches the two ways a run actually exists: * - the cele2e CLI and anything whose argv names it (`cele2e run|up|down|...`) * - a direct `bun test` of this package's suites, which manages the shared * stack through ensureSharedInfra but has no cele2e in argv * The bun-test arm requires bun AND e2e AND test in the command, so an * unrelated `bun test` elsewhere only false-matches when its path names e2e — * and a false match refuses (cheap), where a false miss reaps a live run * (expensive). An editor with an e2e test file open does not match: it does * not start with a bun invocation. */ export function looksLikeE2eRunCommand(command: string): boolean { if (command.includes('cele2e')) return true; return ( /(^|[\\/])bun(\.exe)?\s/.test(command) && command.includes('e2e') && command.includes('test') ); } /** pids of this process's ancestors, self included, bounded at 64 levels. */ function familyOfSelf(): Set { const family = new Set([process.pid]); let pid: number | undefined = process.ppid; for (let i = 0; pid !== undefined && pid > 1 && i < 64; i++) { family.add(pid); try { const out = execFileSync('ps', ['-o', 'ppid=', '-p', String(pid)], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], }).trim(); const ppid = Number.parseInt(out, 10); pid = Number.isFinite(ppid) ? ppid : undefined; } catch { pid = undefined; } } return family; } /** * The real probe: one `ps` listing, filtered by looksLikeE2eRunCommand, with * this process and its ancestry removed — the cleanup runs INSIDE the run it * would otherwise see as evidence, and the run's own launcher shell carries * the same strings in its argv. */ export const realProcessProbe: ProcessProbe = (): number[] => { let listing: string; try { listing = execFileSync('ps', ['-axo', 'pid=,command='], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], }); } catch { // A process table that cannot be read is missing evidence. The caller // treats any probe failure conservatively; here that means reporting no // matches, which the shared-only path then backstops with the age // threshold and the unreadable-age refusal. return []; } const family = familyOfSelf(); const pids: number[] = []; for (const line of listing.split('\n')) { const trimmed = line.trim(); if (!trimmed) continue; const sep = trimmed.indexOf(' '); if (sep <= 0) continue; const pid = Number.parseInt(trimmed.slice(0, sep), 10); if (!Number.isFinite(pid) || family.has(pid)) continue; if (!looksLikeE2eRunCommand(trimmed.slice(sep + 1))) continue; // celilo#1320: a unit-test run matches the command probe but manages no // Docker. An unreadable environ keeps the match — inconclusive never // reaps. if (isUnitOnlyProcess(pid)) continue; pids.push(pid); } return pids; }; /** * Every refusal goes through here so the printed reason is uniformly * greppable as an infrastructure refusal (celilo#1314 direction 3) rather * than reading as a check failure. */ function makeRefusal( detail: string, runningContainers: string[], holder: LockHolder | null, ): LiveStackRefusal { const reason = `[infra-refusal] refusing to clean up: ${detail}\n(this is an environment problem, not a test failure — the run exits 3)`; return { reason, runningContainers, holder }; } /** * The remedy printed with a live-stack refusal. A raw docker removal, because * `cele2e down` does not exist on the hosts that print this message: forgejo * job workspaces are ephemeral, so the binary only exists inside a checkout * (celilo#1314 direction 2). Removing the containers is what unblocks the * guard; the next cleanup sweeps whatever name-prefix resources survive. */ export const CLEAR_STACK_COMMAND = 'docker rm -f $(docker ps -aq --filter name=celilo-e2e)'; /** Docker's `{{.CreatedAt}}` format, e.g. "2026-09-07 18:49:14 +0000 UTC". */ function parseDockerCreatedAt(raw: string | undefined): Date | null { if (raw === undefined) return null; const m = /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2})(?:\.(\d+))? ([+-])(\d{2})(\d{2})/.exec( raw.trim(), ); if (!m) return null; const [, date, time, frac, sign, offH, offM] = m; const utcMs = Date.parse(`${date}T${time}Z`); if (Number.isNaN(utcMs)) return null; const offsetMs = (sign === '-' ? -1 : 1) * (Number(offH) * 60 + Number(offM)) * 60_000; const fracMs = frac ? Number(frac.padEnd(3, '0').slice(0, 3)) : 0; return new Date(utcMs - offsetMs + fracMs); } /** * Containers of the shared compose project. Compose names them * `__` (or with `-` separators on newer compose), so the * character right after the project name is the tell. Everything else live — * per-test projects AND sim-created guests, whose names carry no project — * is run evidence. */ function isSharedStackContainer(name: string): boolean { if (!name.startsWith(SHARED_PROJECT_NAME)) return false; const sep = name[SHARED_PROJECT_NAME.length]; return sep === '_' || sep === '-'; } /** * Container states that mean the container cannot be holding a live stack. * Everything else (running, paused, restarting) is live: paused containers * keep their memory, restarting ones own their networks, and `docker rm -f` * on either kills real work. */ const DEAD_STATES = new Set(['exited', 'created', 'dead']); function isOwnHolder(h: LockHolder): boolean { return h.pid === process.pid || isSameSession(h); } /** * Inspect the machine for a live e2e stack. Returns a refusal when the caller * must not remove anything, null when the environment is provably dead. * * The lock check exempts our own process (the caller holds the lock for its * whole run; without the exemption every run would refuse its own cleanup). * The container check has NO exemption: a stolen lock (the #1297 suspected * path) makes the lock file say "ours" while another run's containers are * still up, so only Docker itself can tell that truth. */ export function findLiveE2eStack( docker: DockerReader = realDocker, lock: () => LockStatus = lockStatus, processes: ProcessProbe = realProcessProbe, ): LiveStackRefusal | null { const status = lock(); if (!status.free && status.holder && !isOwnHolder(status.holder)) { const h = status.holder; const heartbeat = h.state === 'running' ? `, heartbeat ${formatAge(heartbeatAgeMs(h))} old` : ''; return makeRefusal( `the run lock is held by another session — ${formatBusy(h)}${heartbeat}`, [], h, ); } const out = docker([ 'ps', '-a', '--filter', 'name=celilo-e2e', '--format', '{{.Names}}\t{{.State}}\t{{.CreatedAt}}', ]); const live = out .split('\n') .filter(Boolean) .map((line) => { const [name, state, createdAt] = line.split('\t'); return { name: name ?? '', state: state ?? '', createdAt: parseDockerCreatedAt(createdAt) }; }) .filter((c) => c.name !== '' && !DEAD_STATES.has(c.state)); if (live.length === 0) return null; // A per-test container (or a guest, which carries no project name at all) // means a run owns this host. Refuse regardless of the shared stack — this // is the #1297 protection, unchanged. const runOwned = live.filter((c) => !isSharedStackContainer(c.name)); if (runOwned.length > 0) { return makeRefusal( `${live.length} live celilo-e2e-* container(s):\n${live.map((c) => ` - ${c.name}`).join('\n')}\nA live stack owns these. If the run is gone and the stack is truly abandoned, clear it with:\n ${CLEAR_STACK_COMMAND}\n(there may be no cele2e binary outside a checkout — celilo#1314)`, live.map((c) => c.name), null, ); } // celilo#1314: ONLY shared-stack containers are live. Refusing here // unconditionally is what wedged the builder — an orphaned shared stack is // the one state the guard could never resolve. Apply the orphan evidence // test instead. Any inconclusive answer refuses. const runners = processes(); if (runners.length > 0) { return makeRefusal( `a shared-only stack is live (${live.length} celilo-e2e-shared-* container(s)), but a run process is still alive (pid ${runners.join(', ')}) — it may be between suites and about to use the stack. If it is truly abandoned, clear it with:\n ${CLEAR_STACK_COMMAND}`, live.map((c) => c.name), null, ); } const youngest = Math.min( ...live.map((c) => { if (c.createdAt === null) return Number.NaN; return c.createdAt.getTime(); }), ); if (!Number.isFinite(youngest)) { return makeRefusal( `a shared-only stack is live (${live.map((c) => c.name).join(', ')}), but its container age cannot be read from docker — the orphan evidence is inconclusive. If it is truly abandoned, clear it with:\n ${CLEAR_STACK_COMMAND}`, live.map((c) => c.name), null, ); } const ageMs = Date.now() - youngest; if (ageMs < SHARED_ORPHAN_MIN_AGE_MS) { return makeRefusal( `a shared-only stack is live (${live.length} celilo-e2e-shared-* container(s)), but its youngest container is only ${formatAge(ageMs)} old — under the ${SHARED_ORPHAN_MIN_AGE_MS / 60_000}m orphan threshold, so it may belong to a run the probe cannot see. If it is truly abandoned, clear it with:\n ${CLEAR_STACK_COMMAND}`, live.map((c) => c.name), null, ); } // Every piece of run evidence is absent, and the stack is old enough that // no live run can be hiding from the probe. Provably dead: reap it. return null; } /** * The startup-cleanup boundary: refuse with exit 3 when a live stack is in * the way, return when the caller may remove. Exiting here (rather than * throwing) is what makes the refusal survive every caller shape: the run * path's bun test child, `cele2e up`, and the build paths all surface a * process exit code without each one needing its own handling. */ export function refuseOnLiveStack( docker: DockerReader = realDocker, lock: () => LockStatus = lockStatus, processes: ProcessProbe = realProcessProbe, ): void { const refusal = findLiveE2eStack(docker, lock, processes); if (!refusal) return; console.error(`\n${refusal.reason}\n`); process.exit(3); }