// Machine-wide admission control for heavy jobs (browser QA matrices, // production builds): a crash-safe FIFO slot queue with no daemon and no // distributed-lock pretensions. State is plain files on one machine. // // Layout, per resource under a caller-supplied base directory: // // //tickets/--.json waiters // //held/.json holders // // Lexical filename order is enqueue order in BOTH directories. Admission is // an atomic rename tickets/ → held/. A waiter may rename only when the live // held count is under capacity AND its ticket is among the first // (capacity − heldCount) live tickets. After renaming it re-lists held/: if // live holders exceed capacity and its file is not among the capacity // lexically-oldest, it renames itself back and rejoins the queue — the // transient over-admission race self-corrects. Any participant prunes entries // whose PID is dead or whose age exceeds the TTL. Release unlinks the held // file; process death leaves a dead-PID file for the next participant to // prune. // // Toolkit tier: this module must not import src/core (layering check). import { randomUUID } from "node:crypto"; import { mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { coordEnv } from "./env.ts"; export interface AdmissionConfig { /** Base directory holding one subdirectory per resource. */ dir: string; /** Resource name, e.g. "browser-qa" or "build". */ resource: string; /** Maximum concurrent holders. */ capacity: number; /** Poll interval while waiting (default 500ms). */ pollMs?: number; /** Entries older than this are pruned regardless of PID (default 6h). */ ttlMs?: number; } export interface AdmissionEntry { pid: number; label: string; created_at: string; /** Set on holders once admitted. */ acquired_at?: string; } export interface AdmissionWaitInfo { /** 1-based position in the ticket queue. */ position: number; holders: AdmissionEntry[]; } export interface AdmissionAcquireOptions { /** Human-readable holder description shown in status listings. */ label: string; /** Maximum wait before AdmissionTimeoutError (default 20 minutes). */ timeoutMs?: number; /** Progress callback, invoked at most once per poll while waiting. */ onWait?: (info: AdmissionWaitInfo) => void; } export interface AdmissionHandle { /** The held entry's filename (diagnostics). */ entry: string; /** Milliseconds spent waiting in the queue. */ waitedMs: number; /** Give the slot back. Safe to call more than once. */ release: () => void; } export class AdmissionTimeoutError extends Error { readonly holders: AdmissionEntry[]; constructor(resource: string, timeoutMs: number, holders: AdmissionEntry[]) { const labels = holders.map((holder) => `${holder.label} (pid ${holder.pid})`).join(", "); const seconds = timeoutMs < 10_000 ? (timeoutMs / 1000).toFixed(1) : String(Math.round(timeoutMs / 1000)); super( `no ${resource} slot became free within ${seconds}s; ` + `current holder(s): ${labels || "none (queue contention)"}`, ); this.name = "AdmissionTimeoutError"; this.holders = holders; } } const DEFAULT_POLL_MS = 500; const DEFAULT_TTL_MS = 6 * 60 * 60 * 1000; const DEFAULT_TIMEOUT_MS = 20 * 60 * 1000; /** An unparseable entry younger than this is left alone (it may be mid-write * by a non-atomic writer); older, it is torn garbage and gets pruned. */ const UNPARSEABLE_GRACE_MS = 60 * 1000; /** Is a PID alive on this machine? EPERM counts as alive; only ESRCH counts * as dead (fail toward respecting holders rather than stealing slots). */ export function pidAlive(pid: number): boolean { try { process.kill(pid, 0); return true; } catch (err: unknown) { // EPERM = exists but not ours = alive. ESRCH (and anything else) = treat // as dead only on ESRCH; unknown errors count as alive (fail toward // respecting holders rather than stealing their slot). const code = (err as NodeJS.ErrnoException).code; if (code === "ESRCH") return false; return true; } } function ticketsDir(config: AdmissionConfig): string { return join(config.dir, config.resource, "tickets"); } function heldDir(config: AdmissionConfig): string { return join(config.dir, config.resource, "held"); } function ensureDirs(config: AdmissionConfig): void { mkdirSync(ticketsDir(config), { recursive: true }); mkdirSync(heldDir(config), { recursive: true }); } function writeEntry(dir: string, name: string, entry: AdmissionEntry): void { const tmp = join(dir, `.${name}.${randomUUID().slice(0, 8)}.tmp`); writeFileSync(tmp, `${JSON.stringify(entry, null, 2)}\n`); renameSync(tmp, join(dir, name)); } interface LiveEntry { name: string; entry: AdmissionEntry; } /** List live entries in lexical (= enqueue) order, pruning dead-PID, expired, * and torn files as a side effect. Every participant prunes, so a crashed * holder's slot frees as soon as anyone else looks. */ function listLive(dir: string, ttlMs: number): LiveEntry[] { let names: string[]; try { names = readdirSync(dir) .filter((name) => name.endsWith(".json")) .sort(); } catch { return []; } const now = Date.now(); const live: LiveEntry[] = []; for (const name of names) { const path = join(dir, name); let entry: AdmissionEntry | undefined; try { const parsed: unknown = JSON.parse(readFileSync(path, "utf8")); if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { const record = parsed as Record; if (typeof record.pid === "number" && typeof record.created_at === "string") { entry = { pid: record.pid, label: typeof record.label === "string" ? record.label : "", created_at: record.created_at, ...(typeof record.acquired_at === "string" ? { acquired_at: record.acquired_at } : {}), }; } } } catch { // fall through: unparseable } if (!entry) { // Torn or foreign file: prune once it is clearly not mid-write. The // enqueue timestamp is the filename prefix, so age is known without // content. const stamp = Number.parseInt(name.slice(0, 13), 10); if (!Number.isFinite(stamp) || now - stamp > UNPARSEABLE_GRACE_MS) { try { unlinkSync(path); } catch { // a peer pruned it first } } continue; } const age = now - Date.parse(entry.created_at); if (!pidAlive(entry.pid) || Number.isNaN(age) || age > ttlMs) { try { unlinkSync(path); } catch { // a peer pruned it first } continue; } live.push({ name, entry }); } return live; } function sleep(ms: number): Promise { return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); } /** * Join the FIFO queue for one resource and resolve once a slot is held. * Throws AdmissionTimeoutError (with the holders snapshot) when no slot * frees within the timeout — the caller decides whether that is fatal. */ export async function acquireAdmission( config: AdmissionConfig, options: AdmissionAcquireOptions, ): Promise { if (!Number.isInteger(config.capacity) || config.capacity < 1) { throw new Error(`admission capacity must be a positive integer, got ${config.capacity}`); } ensureDirs(config); const pollMs = config.pollMs ?? DEFAULT_POLL_MS; const ttlMs = config.ttlMs ?? DEFAULT_TTL_MS; const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; const tickets = ticketsDir(config); const held = heldDir(config); const myEntry: AdmissionEntry = { pid: process.pid, label: options.label, created_at: new Date().toISOString(), }; const myName = `${String(Date.now()).padStart(13, "0")}-${process.pid}-${randomUUID().slice(0, 8)}.json`; writeEntry(tickets, myName, myEntry); const startedWaiting = Date.now(); const deadline = startedWaiting + timeoutMs; try { while (true) { const liveHeld = listLive(held, ttlMs); const liveTickets = listLive(tickets, ttlMs); const myIndex = liveTickets.findIndex((ticket) => ticket.name === myName); if (myIndex === -1) { // Pruned by a peer (should not happen while we are alive) or lost to // a race; re-enqueue at the back rather than failing the caller. writeEntry(tickets, myName, myEntry); await sleep(pollMs); continue; } const free = config.capacity - liveHeld.length; if (free > 0 && myIndex < free) { renameSync(join(tickets, myName), join(held, myName)); // Verify: the rename can transiently over-admit when a slot freed // between two waiters' listings. Lexical order (= enqueue order) // decides who stays; the loser rejoins the queue with its original // priority because the filename keeps its enqueue timestamp. const heldNow = listLive(held, ttlMs); if (heldNow.length > config.capacity) { const keepers = new Set( heldNow.slice(0, config.capacity).map((holderEntry) => holderEntry.name), ); if (!keepers.has(myName)) { renameSync(join(held, myName), join(tickets, myName)); await sleep(pollMs); continue; } } writeEntry(held, myName, { ...myEntry, acquired_at: new Date().toISOString() }); let released = false; return { entry: myName, waitedMs: Date.now() - startedWaiting, release: () => { if (released) return; released = true; try { unlinkSync(join(held, myName)); } catch { // already pruned } }, }; } options.onWait?.({ position: myIndex + 1, holders: liveHeld.map((holderEntry) => holderEntry.entry), }); if (Date.now() >= deadline) { throw new AdmissionTimeoutError( config.resource, timeoutMs, liveHeld.map((holderEntry) => holderEntry.entry), ); } await sleep(pollMs); } } finally { // On any non-admitted exit (timeout, caller abort via throw), leave no // ticket behind. A successful acquire has already moved the file out. try { unlinkSync(join(tickets, myName)); } catch { // admitted (file moved) or already pruned } } } export interface AdmissionStatus { resource: string; capacity?: number; holders: AdmissionEntry[]; waiters: AdmissionEntry[]; } /** Snapshot the queue for one resource, pruning dead entries on the way. */ export function admissionStatus( config: Pick, ): AdmissionStatus { const ttlMs = config.ttlMs ?? DEFAULT_TTL_MS; const full: AdmissionConfig = { ...config, capacity: 1 }; return { resource: config.resource, holders: listLive(heldDir(full), ttlMs).map((holderEntry) => holderEntry.entry), waiters: listLive(ticketsDir(full), ttlMs).map((ticket) => ticket.entry), }; } /** * Base directory for admission queue state. tmpdir clears on reboot, which * is correct — stale queue state must not outlive the boot. Override with * HARNERY_ADMISSION_DIR. Host CLIs and harn commands must share this helper * so they join the same machine-wide queues. */ export function admissionBaseDir(): string { return coordEnv("ADMISSION_DIR") ?? join(tmpdir(), "harnery-admission"); } /** Resources present under an admission base directory. */ export function listAdmissionResources(dir: string): string[] { try { return readdirSync(dir, { withFileTypes: true }) .filter((entry) => entry.isDirectory()) .map((entry) => entry.name) .sort(); } catch { return []; } }