import * as crypto from "node:crypto"; import type { Stats } from "node:fs"; import * as fs from "node:fs/promises"; import * as path from "node:path"; import { hasFsCode, isEnoent } from "@gajae-code/utils/fs-error"; export interface FileLockOptions { staleMs?: number; retries?: number; retryDelayMs?: number; signal?: AbortSignal; onAcquired?: () => void; /** Stable host identity required to safely reclaim locks on a shared volume. */ ownerHostId?: string; /** Previous local host identities accepted only when deciding stale-owner reclamation. */ previousOwnerHostIds?: readonly string[]; } const DEFAULT_OPTIONS: Required< Omit > = { staleMs: 10_000, retries: 50, retryDelayMs: 100, }; type LockInfo = FileLockOwnerToken; export const FileLockTestHooks: { afterParentMkdir?: (lockPath: string) => void | Promise; } = {}; /** * Returns the OS-provided process start timestamp for PID-reuse detection. * `ps` is available on the supported Unix hosts (macOS and Linux), unlike * Linux's `/proc//stat` pseudo-file. */ export function processStartTime(pid: number): string | null { try { const result = Bun.spawnSync(["ps", "-o", "lstart=", "-p", String(pid)], { stdout: "pipe", stderr: "ignore" }); if (result.exitCode !== 0) return null; const startTime = new TextDecoder().decode(result.stdout).trim(); return startTime || null; } catch { return null; } } let ownProcessStartTime: string | undefined; function currentProcessStartTime(): string { if (ownProcessStartTime === undefined) ownProcessStartTime = processStartTime(process.pid) ?? "unknown"; return ownProcessStartTime; } function cachedProcessStartTime(owner: FileLockOwnerToken, cache?: Map): string | null { if (!cache) return processStartTime(owner.pid); const key = `${owner.pid}:${owner.start_time ?? ""}`; const cached = cache.get(key); if (cached !== undefined || cache.has(key)) return cached ?? null; const startTime = processStartTime(owner.pid); cache.set(key, startTime); return startTime; } function ownerIsAlive(owner: FileLockOwnerToken, startTimeCache?: Map): boolean { if (ownerLiveness(owner.pid) !== "alive") return false; if (!owner.start_time) return true; const currentStartTime = cachedProcessStartTime(owner, startTimeCache); return currentStartTime === null || currentStartTime === owner.start_time; } function lockInfo(ownerHostId?: string): LockInfo { return { pid: process.pid, start_time: currentProcessStartTime(), timestamp: Date.now(), ...(ownerHostId === undefined ? {} : { owner_host_id: ownerHostId }), }; } function writeLockInfo(lockPath: string, info: LockInfo): Promise { return Bun.write(`${lockPath}/info`, JSON.stringify(info)).then(() => info); } async function readLockInfo(lockPath: string): Promise { let parsed: unknown; try { parsed = JSON.parse(await fs.readFile(`${lockPath}/info`, "utf-8")); } catch (error) { if (isEnoent(error) || error instanceof SyntaxError) return null; throw error; } if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null; const { pid, start_time, timestamp, owner_host_id } = parsed as Partial; if ( typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0 || typeof timestamp !== "number" || !Number.isFinite(timestamp) || (start_time !== undefined && (typeof start_time !== "string" || !start_time)) || (owner_host_id !== undefined && (typeof owner_host_id !== "string" || !owner_host_id)) ) return null; return { pid, start_time, timestamp, owner_host_id }; } /** @internal */ export async function readFileLockInfoForGc(lockDir: string): Promise { return await readLockInfo(lockDir); } /** Owner identity stamped into a `.lock/info` record. */ export interface FileLockOwnerToken { pid: number; start_time?: string; owner_host_id?: string; timestamp: number; } function getLockPath(filePath: string): string { return `${filePath}.lock`; } /** Outcome of a guarded lock-dir removal attempt (`removeFileLockDirForGc`). */ export type FileLockGcRemoval = "removed" | "owner_changed" | "missing"; interface LockDirStatToken { dev: number; ino: number; mtimeMs: number; ctimeMs: number; } type LockStaleSnapshot = | { stale: false } | { stale: true; owner: FileLockOwnerToken } | { stale: true; owner: null; stat: LockDirStatToken }; /** * @internal * Fail-closed removal of a lock dir whose owner is expected to be dead or * finished. Re-reads the on-disk owner token as close to the unlink as possible * and only deletes the dir when it STILL holds the exact `{pid, timestamp}` * identity the caller observed. * * Closes stale-cleanup TOCTOU windows (#606): between a dead/stale re-read and * the unlink, a live process can reclaim a stale lock at the same path * (`acquireLock` rms the stale dir, then re-`mkdir`s and rewrites `info` with a * fresh pid+timestamp). Deleting by path alone would reap that LIVE lock. Any * mismatch (`owner_changed`) or absent/unreadable info (`missing` — e.g. a * fresh acquirer between `mkdir` and `writeLockInfo`) refuses the delete and * leaves the dir intact. POSIX has no atomic compare-and-delete for a * directory, so the residual read->unlink window cannot be fully eliminated, * but the reclaim-after-stale scenario the issue describes is now guarded. */ export async function removeFileLockDirForGc( lockDir: string, expected: FileLockOwnerToken, ): Promise { const current = await readLockInfo(lockDir); if (!current) return "missing"; if ( current.pid !== expected.pid || (expected.start_time !== undefined && current.start_time !== expected.start_time) || current.owner_host_id !== expected.owner_host_id || current.timestamp !== expected.timestamp ) { return "owner_changed"; } await fs.rm(lockDir, { recursive: true, force: true }); return "removed"; } type OwnerLiveness = "alive" | "dead" | "unknown"; function ownerLiveness(pid: number): OwnerLiveness { if (!Number.isFinite(pid) || pid <= 0) return "unknown"; try { process.kill(pid, 0); return "alive"; } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === "ESRCH") return "dead"; // EPERM means the process exists but we may not signal it; treat as alive. // Anything else is indeterminate. return code === "EPERM" ? "alive" : "unknown"; } } function statToken(stats: Stats): LockDirStatToken { return { dev: stats.dev, ino: stats.ino, mtimeMs: stats.mtimeMs, ctimeMs: stats.ctimeMs, }; } function sameStatToken(a: LockDirStatToken, b: LockDirStatToken): boolean { return a.dev === b.dev && a.ino === b.ino && a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs; } async function staleLockSnapshot( lockPath: string, staleMs: number, ownerHostId?: string, previousOwnerHostIds: readonly string[] = [], startTimeCache?: Map, ): Promise { let info: LockInfo | null; try { info = await readLockInfo(lockPath); } catch (error) { // Windows can transiently deny reads of a just-created lock metadata file // while another contender is publishing it. Treat that as active // contention and retry rather than failing the caller or reaping by path. if (hasFsCode(error, "EPERM")) return { stale: false }; throw error; } if (!info && ownerHostId !== undefined) return { stale: false }; if (!info) { try { const stats = await fs.stat(lockPath); if (Date.now() - stats.mtimeMs <= staleMs) return { stale: false }; return { stale: true, owner: null, stat: statToken(stats) }; } catch (err) { if (isEnoent(err)) return { stale: false }; throw err; } } // A host-qualified lock may only be reclaimed after proving that its owner is // local. Foreign and malformed host-qualified records fail closed: PID values // and clocks are not meaningful across hosts. if ( ownerHostId !== undefined && info.owner_host_id !== ownerHostId && !previousOwnerHostIds.includes(info.owner_host_id ?? "") ) return { stale: false }; // Never reap a live owner by elapsed time: a long legitimate critical section must // not have its lock stolen (#652). Reclaim a dead owner immediately. Only when owner // liveness is indeterminate do we fall back to the staleMs elapsed-time heuristic. if (ownerIsAlive(info, startTimeCache)) return { stale: false }; if (ownerLiveness(info.pid) === "dead" || Date.now() - info.timestamp > staleMs) { return { stale: true, owner: info }; } return { stale: false }; } async function removeStaleLockForAcquire(lockPath: string, snapshot: LockStaleSnapshot): Promise { if (!snapshot.stale) return false; if (snapshot.owner) { return (await removeFileLockDirForGc(lockPath, snapshot.owner)) === "removed"; } const currentInfo = await readLockInfo(lockPath); if (currentInfo) return false; try { const currentStats = await fs.stat(lockPath); if (!sameStatToken(statToken(currentStats), snapshot.stat)) return false; await fs.rm(lockPath, { recursive: true, force: true }); return true; } catch (err) { if (isEnoent(err)) return false; throw err; } } /** * @internal * READ-ONLY verdict on an EXISTING generic `.lock/` directory that another lock * protocol has collided with: is its owner gone, by this protocol's own rules? * * Exposed so a foreign holder of the same path never has to reimplement this protocol's * owner parsing or liveness rules. Reusing them is what makes the two implementations * agree: `processStartTime` here is the portable `ps` value that `info.start_time` was * written from, so a live owner is proved live rather than compared against a value from a * different clock source and then reaped. A live owner is never reported stale by elapsed * time alone. * * Deletion is deliberately NOT offered. This protocol can only re-read an owner token and * then unlink a pathname, which a successor can take over in between; a caller that must * remove the directory has to do it under an identity-bound primitive that refuses when * the object is no longer the one that was judged. */ export async function genericFileLockDirIsStale( lockDir: string, staleMs: number, ownerHostId?: string, ): Promise { return (await staleLockSnapshot(lockDir, staleMs, ownerHostId)).stale; } async function tryAcquireLock( lockPath: string, ownerHostId?: string, onAcquired?: () => void, ): Promise { await fs.mkdir(path.dirname(lockPath), { recursive: true }); const afterParentMkdir = FileLockTestHooks.afterParentMkdir; if (afterParentMkdir) await afterParentMkdir(lockPath); if (ownerHostId === undefined) { try { await fs.mkdir(lockPath); onAcquired?.(); return await writeLockInfo(lockPath, lockInfo()); } catch (error) { if ((error as NodeJS.ErrnoException).code === "EEXIST") return null; throw error; } } const pendingPath = `${lockPath}.pending.${process.pid}.${crypto.randomUUID()}`; const owner = lockInfo(ownerHostId); try { await fs.mkdir(pendingPath); await writeLockInfo(pendingPath, owner); try { await fs.rename(pendingPath, lockPath); onAcquired?.(); return owner; } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === "EEXIST" || code === "ENOTEMPTY") return null; if (code === "EPERM") { try { await fs.stat(lockPath); return null; } catch (statError) { if (!isEnoent(statError)) throw statError; } } throw error; } } finally { await fs.rm(pendingPath, { recursive: true, force: true }).catch(() => undefined); } } async function releaseLock(lockPath: string, owner: FileLockOwnerToken): Promise { const outcome = await removeFileLockDirForGc(lockPath, owner); if (outcome !== "removed") throw new Error(`Failed to release file lock: ${outcome}.`); } /** * Bounded, actionable description of who holds `lockPath` at exhaustion time. * Never a stealing authority: purely diagnostic, read once after the last retry. */ async function lockHolderDescription(lockPath: string): Promise { try { const info = await readLockInfo(lockPath); if (info) { // A lock record carrying a foreign owner_host_id belongs to another // machine (shared-volume topic registry): its pid is meaningful only // on that host, so probing the same numeric pid here could mislabel a // coincident local process as the holder. Report the owner host with // unknown liveness instead. if (info.owner_host_id !== undefined) { return ( `held by pid ${info.pid} on host ${info.owner_host_id} (liveness unknown from this host)` + ` since ${new Date(info.timestamp).toISOString()}` ); } // Same-host holder: use the full liveness proof (pid alive AND, when the // record carries a start_time, the start-time identity match) so a dead // holder whose pid was already reused is not mislabeled "(live)". const alive = ownerIsAlive(info); return ( `held by pid ${info.pid}` + (alive ? " (live)" : ownerLiveness(info.pid) === "dead" ? " (dead but not reaped)" : " (liveness unknown)") + ` since ${new Date(info.timestamp).toISOString()}` ); } try { await fs.stat(path.join(lockPath, "info")); return "held by an owner whose metadata is not yet readable"; } catch (error) { if (!isEnoent(error)) throw error; } return "held by an unrecognized owner record"; } catch (error) { return `held by an unreadable owner (${(error as Error).message})`; } } async function acquireLock(filePath: string, options: FileLockOptions = {}): Promise<() => Promise> { if (options.ownerHostId !== undefined && !options.ownerHostId) throw new Error("ownerHostId must be non-empty"); if (options.previousOwnerHostIds?.some(hostId => !hostId)) throw new Error("previousOwnerHostIds must contain only non-empty identities"); const opts = { ...DEFAULT_OPTIONS, ...options }; const lockPath = getLockPath(filePath); const contentionStartTimes = new Map(); for (let attempt = 0; attempt < opts.retries; attempt++) { if (opts.signal?.aborted) throw opts.signal.reason ?? new Error("File lock acquisition aborted"); const owner = await tryAcquireLock(lockPath, opts.ownerHostId, opts.onAcquired); if (owner) return () => releaseLock(lockPath, owner); const stale = await staleLockSnapshot( lockPath, opts.staleMs, opts.ownerHostId, opts.previousOwnerHostIds, contentionStartTimes, ); if (await removeStaleLockForAcquire(lockPath, stale)) continue; if (!opts.signal) { await Bun.sleep(opts.retryDelayMs); continue; } const { promise, resolve, reject } = Promise.withResolvers(); const onAbort = (): void => reject(opts.signal?.reason ?? new Error("File lock acquisition aborted")); opts.signal.addEventListener("abort", onAbort, { once: true }); void Bun.sleep(opts.retryDelayMs).then(resolve); try { await promise; } finally { opts.signal.removeEventListener("abort", onAbort); } } throw new Error( `Failed to acquire lock for ${filePath} after ${opts.retries} attempts: ${await lockHolderDescription(lockPath)} (${lockPath}); ` + `a live owner is never displaced — if this is an SDK broker (gjc sdk status), it must finish or be stopped before retrying`, ); } /** * Serializes all contenders, including callers in the same process. Because this * API exposes no ownership token, recursive acquisition is indistinguishable * from independent async contention; code that already holds the lock must pass * that fact through its own `lockHeld` path instead of acquiring it again. */ export async function withFileLock( filePath: string, fn: () => Promise, options: FileLockOptions = {}, ): Promise { const release = await acquireLock(filePath, options); let result: T; try { result = await fn(); } catch (operationError) { try { await release(); } catch (releaseError) { throw new AggregateError([operationError, releaseError], "File lock operation and release both failed."); } throw operationError; } await release(); return result; }