/** * cele2e run-lock — cross-platform mutual exclusion for the shared e2e infra. * * cele2e is a single shared resource: the shared infra is one global compose * project (celilo-e2e-shared) and start-of-run cleanup tears down every * celilo-e2e-* container regardless of who started it. So two actors at once * (two Claude sessions, or a session + an operator at the terminal) clobber * each other. This lock serializes runs across processes — no external tools, * identical on macOS and Linux. * * Design (Forgejo #243): * - Acquire via fs.openSync(path, 'wx') — POSIX-atomic exclusive create, * same behavior on mac+linux. The lockfile lives at a MACHINE-GLOBAL path * (~/.cache/celilo-e2e/run.lock), NOT under any repo/worktree, because the * Docker infra is global regardless of which checkout started it. * - Fail fast on contention with a message naming the holder. Sessions poll * `cele2e status` and decide for themselves whether to wait (no --wait). * - Staleness: on the SAME host, PID-liveness is authoritative (process.kill * (pid, 0)); a dead holder PID → reclaim. Cross-host we can't check the PID, * so fall back to a heartbeat TTL (beatAt older than STALE_TTL_MS). * - A `kept` lock (left by `--keep` / `up`) is never reclaimed by ANOTHER * actor — it guards a stack that outlives the process. Its OWN session * reclaims it automatically (see isSameSession); everyone else clears it * deliberately via `cele2e release` / `cele2e down`. * - SUSPECT holders: a live PID whose heartbeat has gone quiet is the one * hang the PID check cannot see (a build wedged mid-step keeps its process * alive and sleeping). isSuspect() names it so `status`/`doctor` can flag it * instead of leaving 20 silent minutes to be found by hand. */ import { execFileSync } from 'node:child_process'; import { closeSync, mkdirSync, openSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; import { homedir, hostname } from 'node:os'; import { basename, dirname, join } from 'node:path'; const STALE_TTL_MS = 90_000; const HEARTBEAT_MS = 30_000; /** * How quiet a RUNNING holder's heartbeat may go before it is suspect. * * Calibrated against what a HEALTHY holder actually does. The heartbeat is a * setInterval, and build-infra shells out with spawnSync — which blocks the * event loop for the entire duration of each `docker build`. So a perfectly * healthy build stops beating for as long as its slowest image takes: a live * holder was observed 67s stale while making normal progress, and the largest * images here are ~2GB. Anything near the 30s tick interval would flag those. * * Ten minutes sits above any single legitimate image build (the per-image * watchdog caps one at 15) and far below the ~20 and ~32 minute hangs that * went unnoticed. Deliberately NOT a reclaim threshold: the PID is alive, and * killing someone's wedged build out from under them is the operator's call. */ export const SUSPECT_HEARTBEAT_MS = 600_000; /** * Machine-global lock path — NOT under any repo/worktree, because the Docker * infra is global regardless of which checkout started it. Overridable via * CELILO_E2E_LOCK_PATH (used by tests; also a handy operator escape hatch). */ export function lockPath(): string { return process.env.CELILO_E2E_LOCK_PATH || join(homedir(), '.cache', 'celilo-e2e', 'run.lock'); } export interface LockHolder { pid: number; hostname: string; session: string; test: string; runId: string; startedAt: string; beatAt: number; state: 'running' | 'kept'; } interface Held { keepOnRelease: boolean; heartbeat: ReturnType | null; released: boolean; } /** Set once this process owns the lock; null otherwise. */ let held: Held | null = null; export class E2eBusyError extends Error { constructor(public holder: LockHolder) { super(formatBusy(holder)); this.name = 'E2eBusyError'; } } /** Human label for who's running, e.g. "e2e busy: running , started 3m ago (pid 1234)". */ export function formatBusy(h: LockHolder): string { const age = ageString(Date.parse(h.startedAt)); const verb = h.state === 'kept' ? 'holds a kept stack from' : 'running'; const what = h.state === 'kept' ? '(run `cele2e release` to free it)' : `${h.test}, started ${age} ago (pid ${h.pid})`; const suspect = isSuspect(h) ? ` — SUSPECT: no heartbeat for ${formatAge(heartbeatAgeMs(h))} (process alive but not progressing)` : ''; return `e2e busy: ${h.session} ${verb} ${what}${suspect}`; } function ageString(since: number): string { return formatAge(Date.now() - since); } /** Human duration, e.g. "45s" / "32m" / "2h05m". */ export function formatAge(ms: number): string { const s = Math.max(0, Math.floor(ms / 1000)); if (s < 60) return `${s}s`; if (s < 3600) return `${Math.floor(s / 60)}m`; return `${Math.floor(s / 3600)}h${String(Math.floor((s % 3600) / 60)).padStart(2, '0')}m`; } /** Branch + worktree path so a holder maps back to a specific session/thread. */ function deriveSession(): string { if (process.env.CELILO_E2E_SESSION) return process.env.CELILO_E2E_SESSION; const cwd = process.cwd(); const git = (args: string[]): string => { try { return execFileSync('git', args, { cwd, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], }).trim(); } catch { return ''; } }; const branch = git(['rev-parse', '--abbrev-ref', 'HEAD']); const top = git(['rev-parse', '--show-toplevel']); const base = top ? basename(top) : basename(cwd); if (branch && top) return `${branch} (${top})`; return branch || base; } /** * Is this holder the caller's own session — same host, same branch+worktree? * This is what separates "my own kept stack is in my way" (friction, auto-clear) * from "someone else is mid-run" (contention, refuse). */ export function isSameSession(h: LockHolder): boolean { return h.hostname === hostname() && h.session === deriveSession(); } /** Milliseconds since the holder last refreshed its heartbeat. */ export function heartbeatAgeMs(h: LockHolder): number { return Math.max(0, Date.now() - h.beatAt); } /** * A live process whose heartbeat has gone quiet — the hang PID-liveness misses. * `kept` holders are excluded: their beatAt is frozen on purpose at release. */ export function isSuspect(h: LockHolder): boolean { return h.state === 'running' && heartbeatAgeMs(h) > SUSPECT_HEARTBEAT_MS; } function pidAlive(pid: number): boolean { try { process.kill(pid, 0); return true; } catch (err) { // ESRCH = no such process (dead). EPERM = alive but not ours (still alive). return (err as NodeJS.ErrnoException).code === 'EPERM'; } } export function readHolder(): LockHolder | null { try { return JSON.parse(readFileSync(lockPath(), 'utf-8')) as LockHolder; } catch { return null; } } /** * Is the on-disk holder stale (safe to reclaim)? A `kept` lock is never stale — * it deliberately outlives its process. On the same host the PID is the source * of truth; cross-host we can only use the heartbeat TTL. */ function isStale(h: LockHolder): boolean { if (h.state === 'kept') return false; if (h.hostname === hostname()) return !pidAlive(h.pid); return Date.now() - h.beatAt > STALE_TTL_MS; } function writeHolder(fd: number, h: LockHolder): void { writeFileSync(fd, JSON.stringify(h, null, 2)); } /** What the acquire had to clear on the way in, so callers can say so out loud. */ export interface AcquireOutcome { /** A `kept` lock left by this same session that we auto-released, else null. */ autoReleasedOwnKept: LockHolder | null; } /** * Acquire the run lock for this process. Throws E2eBusyError if another live * (non-stale) holder owns it. Registers an exit handler so the lock is released * on any process.exit() path (normal, exception, or a SIGINT handler that * exits). A signal-killed process with no exit handler leaks the lock, but the * next run reclaims it via PID-liveness — that's exactly what staleness is for. * * A `kept` lock left by THIS session (same host, same branch+worktree) is * auto-released and reported in the outcome. Refusing there protected nobody: * the only stack at risk was the caller's own, and the refusal was routinely * misread as a finished run — the previous run's results dir is still sitting * there looking like a clean pass. Another session's kept lock still refuses. */ export function acquireRunLock(opts: { test: string; runId: string; allowKept?: boolean; }): AcquireOutcome { mkdirSync(dirname(lockPath()), { recursive: true }); let autoReleasedOwnKept: LockHolder | null = null; const holder: LockHolder = { pid: process.pid, hostname: hostname(), session: deriveSession(), test: opts.test, runId: opts.runId, startedAt: new Date().toISOString(), beatAt: Date.now(), state: 'running', }; for (let attempt = 0; attempt < 2; attempt++) { let fd: number; try { fd = openSync(lockPath(), 'wx'); } catch (err) { if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; const existing = readHolder(); // Reclaim when: corrupt/unreadable, the holder is stale, this is a // `--reuse` run taking over a `kept` stack it's deliberately reusing, OR // the `kept` stack belongs to this very session (see the doc comment). const ownKept = !!existing && existing.state === 'kept' && isSameSession(existing); if ( !existing || isStale(existing) || (opts.allowKept && existing.state === 'kept') || ownKept ) { if (ownKept && existing) autoReleasedOwnKept = existing; try { unlinkSync(lockPath()); } catch {} continue; } throw new E2eBusyError(existing); } writeHolder(fd, holder); closeSync(fd); const heartbeat = setInterval(() => { // Only refresh while running; a kept lock's beatAt is frozen by design. if (!held || held.released || held.keepOnRelease) return; try { const cur = readHolder(); if (cur && cur.pid === process.pid) { writeFileSync(lockPath(), JSON.stringify({ ...cur, beatAt: Date.now() }, null, 2)); } } catch {} }, HEARTBEAT_MS); heartbeat.unref(); held = { keepOnRelease: false, heartbeat, released: false }; process.on('exit', releaseRunLock); return { autoReleasedOwnKept }; } // Two reclaim attempts both lost the race → someone else won fair and square. const existing = readHolder(); throw new E2eBusyError(existing ?? holderUnknown()); } function holderUnknown(): LockHolder { return { pid: 0, hostname: hostname(), session: 'unknown', test: '?', runId: '?', startedAt: new Date().toISOString(), beatAt: Date.now(), state: 'running', }; } /** * Mark the lock to survive process exit in a `kept` state — call when a stack * (network/containers) will outlive this process (`--keep`, `up`). The next run * then refuses to clobber it until `cele2e release` / `cele2e down`. */ export function markKept(): void { if (held) held.keepOnRelease = true; } /** Release the lock. Idempotent. Honors markKept() (kept-state) vs delete. */ export function releaseRunLock(): void { if (!held || held.released) return; held.released = true; if (held.heartbeat) clearInterval(held.heartbeat); if (held.keepOnRelease) { const cur = readHolder(); if (cur && cur.pid === process.pid) { try { writeFileSync( lockPath(), JSON.stringify({ ...cur, state: 'kept', beatAt: Date.now() }, null, 2), ); } catch {} } return; } try { const cur = readHolder(); if (!cur || cur.pid === process.pid) unlinkSync(lockPath()); } catch {} } /** Forcibly clear the lock regardless of holder — used by `cele2e release`/`down`. */ export function clearLock(): boolean { try { unlinkSync(lockPath()); return true; } catch { return false; } } export interface LockStatus { free: boolean; holder: LockHolder | null; /** Milliseconds since the holder's last heartbeat; null when free. */ heartbeatAgeMs: number | null; /** Live PID, quiet heartbeat — the hang the PID check cannot see. */ suspect: boolean; /** The holder is this session's own kept stack, which the next run reclaims. */ ownKept: boolean; } /** Current lock state for `cele2e status` (and any poller). */ export function lockStatus(): LockStatus { const h = readHolder(); if (!h || isStale(h)) { return { free: true, holder: null, heartbeatAgeMs: null, suspect: false, ownKept: false }; } return { free: false, holder: h, heartbeatAgeMs: heartbeatAgeMs(h), suspect: isSuspect(h), ownKept: h.state === 'kept' && isSameSession(h), }; }