/** * Shared runtime lifecycle foundation. * * Two minimal, deliberately small primitives that subsystem runtimes * (DAP/LSP/MCP stdio, eval workers, etc.) adopt so spawned children and * non-process resources cannot outlive their owner: * * F1(a) `spawnOwnedProcess` — wraps `ptree.spawn` with explicit * process-group ownership, escalating (SIGTERM -> grace -> SIGKILL) * tree termination, bounded `awaitExit`, abort-listener cleanup on * settle, idempotent `dispose`, and a single postmortem hook that * reaps every still-live owned process group on fatal/normal shutdown. * * F1(b) `registerResourceOwner` — a generic, idempotent postmortem adapter * for non-process resources (Bun Workers, VM contexts, timers, * sockets) built on the existing `postmortem.register` facility. * * Ownership is keyed to the *process group*, not the root process. A root that * exits after backgrounding descendants (`sh -c "worker & exit 0"`) keeps the * owner registered until the group is actually gone, so the descendant tree is * still reaped by `dispose()`/postmortem. * * This module intentionally owns only these primitives. It does not migrate * existing call sites; subsystem PRs adopt it incrementally. * * Note: `ptree.spawn` always pipes stdout/stderr. Adopters that expect output * (DAP/LSP/MCP protocol servers) must consume `owner.child.stdout`; F1 does not * drain it, so a chatty child whose stdout is never read can still block on a * full pipe. That draining is the adopter's responsibility. */ import * as fs from "node:fs"; import { logger, postmortem, ptree } from "@gajae-code/utils"; import { type LinuxProcPidProbeResult, probeLinuxProcPidSync } from "../gjc-runtime/linux-proc"; const DEFAULT_GRACEFUL_MS = 2_000; // Hard cap for how long `dispose()` waits after SIGKILL before giving up so a // wedged, unkillable child can never block shutdown forever. const SIGKILL_REAP_CAP_MS = 2_000; // After the root process exits on its own, how long to wait for the process // group to drain before deregistering. Clean servers drain immediately; a root // that backgrounded descendants stays registered past this window. const ROOT_EXIT_DRAIN_MS = 250; const isPosix = process.platform !== "win32"; const delay = (ms: number): Promise => new Promise(resolve => { const timer = setTimeout(resolve, Math.max(0, ms)); timer.unref?.(); }); /** Poll `predicate` until it is true or `timeoutMs` elapses. Returns the final value. */ async function pollUntil(predicate: () => boolean, timeoutMs: number, intervalMs = 20): Promise { if (predicate()) return true; const deadline = Date.now() + Math.max(0, timeoutMs); while (Date.now() < deadline) { await delay(Math.min(intervalMs, Math.max(0, deadline - Date.now()))); if (predicate()) return true; } return predicate(); } /** Whether a POSIX process group still has any member (zombies count as alive). */ function groupAlive(pgid: number): boolean { try { process.kill(-pgid, 0); return true; } catch (err) { // EPERM => the group exists but we cannot signal it; treat as alive. return (err as NodeJS.ErrnoException).code === "EPERM"; } } /** Whether an unreadable `/proc` entry might still describe a running process. */ export function procEntryMayStillBeRunning(error: unknown): boolean { const code = (error as NodeJS.ErrnoException).code; return code !== "ENOENT" && code !== "ESRCH"; } /** Whether a process-group leader still has the spawn-time identity we own. */ export function groupLeaderIdentityMatches( expectedStartTime: string | undefined, leader: LinuxProcPidProbeResult, ): boolean { // An absent leader cannot be a recycled leader. Its group may still contain // descendants that we own, including when the root exits before we can read // its start time immediately after spawn. if (leader.kind === "absent") return true; return expectedStartTime !== undefined && leader.kind === "live" && leader.startTime === expectedStartTime; } /** * Whether a POSIX process group still has a *running* member. Zombies are * inert: they execute no code and can only be reaped by their parent, which * the owned process cannot control. Teardown must not wait on an external * reaper, so disposal considers a group terminated once every member is dead * or a zombie. On Linux this inspects `/proc`; elsewhere it conservatively * falls back to {@link groupAlive} (zombies count as alive), so teardown there * relies on the bounded SIGTERM/SIGKILL windows rather than zombie detection. */ function groupHasRunningMembers(pgid: number): boolean { if (!groupAlive(pgid)) return false; if (process.platform !== "linux") return true; try { for (const entry of fs.readdirSync("/proc")) { if (entry.length === 0 || entry.charCodeAt(0) < 0x30 || entry.charCodeAt(0) > 0x39) continue; let stat: string; try { stat = fs.readFileSync(`/proc/${entry}/stat`, "utf8"); } catch (error) { // Entries disappearing during enumeration are expected. Every other // failure leaves the member's state unknowable, so fail closed. if (!procEntryMayStillBeRunning(error)) continue; return true; } // Format: pid (comm) state ppid pgrp session tty ...; comm may contain // spaces or parens, so parse everything after the last ')'. const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); const state = fields[0] ?? ""; if (Number(fields[2]) === pgid && state !== "Z" && state !== "X") { return true; } } return false; } catch { // /proc unavailable or unreadable: keep the conservative behavior. return true; } } /** Options for {@link spawnOwnedProcess}. */ export interface SpawnOwnedOptions { cwd?: string; env?: Record; /** stdin mode passed through to the child. Defaults to `"ignore"`. */ stdin?: "pipe" | "ignore"; /** When aborted, the owned process tree is disposed (escalating kill). */ signal?: AbortSignal; /** Grace period (ms) between SIGTERM and SIGKILL on dispose. Default 2000. */ gracefulMs?: number; /** * Spawn the child as its own process-group leader so the whole descendant * tree can be signalled on dispose. Defaults to `true` on POSIX. Has no * effect on Windows, where teardown falls back to single-process kill. */ processGroup?: boolean; /** Label used in diagnostics. */ name?: string; } /** Result of a bounded {@link OwnedProcess.awaitExit}. */ export interface AwaitExitResult { /** `true` when the process has exited; `false` when the timeout fired first. */ exited: boolean; /** Exit code if known, else `null`. */ code: number | null; } /** The observed outcome of an owned-process teardown attempt. */ export type OwnedProcessTeardownStatus = "terminated" | "still_running" | "identity_unverified"; /** Result of {@link OwnedProcess.dispose}. */ export interface OwnedProcessTeardownResult { status: OwnedProcessTeardownStatus; } /** A spawned child process owned by the runtime with guaranteed teardown. */ export interface OwnedProcess { readonly child: ptree.ChildProcess; readonly pid: number | undefined; /** Resolves/rejects when the root child exits (mirrors ptree's `exited`). */ readonly exited: Promise; /** `true` once `dispose()` has started. */ readonly disposed: boolean; /** * Wait for the root child to exit, optionally bounded by `timeoutMs`. With no * timeout it resolves only when the child exits. Never rejects. */ awaitExit(opts?: { timeoutMs?: number }): Promise; /** * Idempotently terminate the owned process *group*: SIGTERM the group, wait * `gracefulMs`, then SIGKILL, polling group liveness throughout. Removes the * abort listener and deregisters from the live-owner set only after teardown * has completed. Repeated/concurrent calls return the same in-flight promise. */ dispose(): Promise; } const liveOwners = new Set(); let ownedPostmortemRegistered = false; function ensureOwnedPostmortem(): void { if (ownedPostmortemRegistered) return; ownedPostmortemRegistered = true; postmortem.register("runtime:owned-processes", async () => { await Promise.all([...liveOwners].map(owner => owner.dispose().catch(() => undefined))); }); } /** * Spawn a child process owned by the runtime. The returned {@link OwnedProcess} * is registered for postmortem cleanup and tears down its whole process group * on dispose/abort. */ export function spawnOwnedProcess(cmd: string[], opts: SpawnOwnedOptions = {}): OwnedProcess { const gracefulMs = opts.gracefulMs ?? DEFAULT_GRACEFUL_MS; const useGroup = (opts.processGroup ?? true) && isPosix; ensureOwnedPostmortem(); // We deliberately do NOT forward `opts.signal` to `ptree.spawn`: ptree's // `attachSignal` only kills the single process, whereas owned teardown must // signal the whole group. We wire our own abort listener below and remove it // on settle so long-lived signals never accumulate listeners. const child = ptree.spawn(cmd, { cwd: opts.cwd, env: opts.env, stdin: opts.stdin ?? "ignore", detached: useGroup, }); // On POSIX with `detached`, the child is its own process-group leader, so the // group id equals its pid. `undefined` => single-process (Windows/opt-out). const pgid = useGroup ? child.pid : undefined; const groupLeader = pgid === undefined ? undefined : probeLinuxProcPidSync(pgid); const groupLeaderStartTime = groupLeader?.kind === "live" ? groupLeader.startTime : undefined; let disposed = false; let disposePromise: Promise | undefined; let deregistered = false; // Terminal once teardown/reconciliation has confirmed the group is gone. A // late dispose() must then be a true no-op and never re-probe a pgid the OS // may have recycled into an unrelated group. let terminated = false; let onAbort: (() => void) | undefined; const removeAbort = (): void => { if (onAbort && opts.signal) { opts.signal.removeEventListener("abort", onAbort); onAbort = undefined; } }; const deregister = (terminal = true): void => { if (deregistered) return; deregistered = true; if (terminal) terminated = true; liveOwners.delete(owner); removeAbort(); }; // Identity verification exists to stop a recycled pgid from being signalled. // It is only decidable where the leader's spawn-time identity is readable // (`/proc`). Elsewhere the group id is still ours for the lifetime of the // owned child, so an unverifiable probe must not be reported as a mismatch — // doing so skips SIGTERM/SIGKILL entirely and leaks the whole child tree. const groupIdentityMatches = (): boolean => { if (pgid === undefined) return false; if (process.platform !== "linux") return true; const leader = probeLinuxProcPidSync(pgid); // Once the original leader has exited, its group can still own descendants. // A replacement group has a leader at this pid, whose start time must match. return groupLeaderIdentityMatches(groupLeaderStartTime, leader); }; const signalTree = (signal: NodeJS.Signals): void => { const pid = child.pid; if (pid === undefined) return; if (pgid !== undefined) { try { // Negative pid signals the entire process group (child is leader). process.kill(-pgid, signal); return; } catch { // Group already gone; nothing to do. } return; } if (signal === "SIGKILL") { try { process.kill(pid, "SIGKILL"); } catch { /* already gone */ } } else { // ptree's kill terminates the single process via the native handle. child.kill(); } }; const owner: OwnedProcess = { child, get pid() { return child.pid; }, get exited() { return child.exited; }, get disposed() { return disposed; }, async awaitExit({ timeoutMs }: { timeoutMs?: number } = {}): Promise { const exitedResult = child.exited .then(code => ({ exited: true as const, code: code as number | null })) .catch(() => ({ exited: true as const, code: child.exitCode })); if (timeoutMs === undefined) return exitedResult; let timer: ReturnType | undefined; const timeout = new Promise(resolve => { timer = setTimeout(() => resolve({ exited: false, code: child.exitCode }), Math.max(0, timeoutMs)); timer.unref?.(); }); try { return await Promise.race([exitedResult, timeout]); } finally { if (timer) clearTimeout(timer); } }, dispose(): Promise { // Already terminal (e.g. clean drain reconciled and deregistered): // never re-probe the pgid; treat dispose as a settled no-op. if (terminated) { disposed = true; if (!disposePromise) disposePromise = Promise.resolve({ status: "terminated" }); return disposePromise; } if (disposePromise) return disposePromise; disposed = true; removeAbort(); const result = (status: OwnedProcessTeardownStatus): OwnedProcessTeardownResult => ({ status }); disposePromise = (async (): Promise => { try { if (pgid !== undefined) { // Group ownership: terminate every *running* member, even if the // root has already exited (it may have backgrounded children). // Liveness is measured by running members, not raw group // existence: SIGTERM'd descendants become zombies that only their // parent (often PID 1) can reap, and teardown must not burn its // grace window waiting on an external reaper. if (!groupAlive(pgid)) return result("terminated"); if (!groupIdentityMatches()) return result("identity_unverified"); signalTree("SIGTERM"); if (await pollUntil(() => !groupHasRunningMembers(pgid), gracefulMs)) return result("terminated"); if (!groupIdentityMatches()) return result("identity_unverified"); signalTree("SIGKILL"); if (await pollUntil(() => !groupHasRunningMembers(pgid), SIGKILL_REAP_CAP_MS)) return result("terminated"); logger.warn("owned process group still running after SIGKILL", { name: opts.name, pgid, }); return result("still_running"); } // Single-process fallback (Windows / processGroup:false). if (child.exitCode !== null) return result("terminated"); signalTree("SIGTERM"); if ((await owner.awaitExit({ timeoutMs: gracefulMs })).exited) return result("terminated"); signalTree("SIGKILL"); return (await owner.awaitExit({ timeoutMs: SIGKILL_REAP_CAP_MS })).exited ? result("terminated") : result("still_running"); } catch (err) { logger.warn("owned process dispose failed", { name: opts.name, error: err instanceof Error ? err.message : String(err), }); return result("still_running"); } })().then(res => { // `identity_unverified` and `still_running` leave the owner registered // so a later postmortem or caller retry can make another bounded attempt. // Only a confirmed `terminated` closes the book. if (res.status === "terminated") { deregister(true); } else { disposePromise = undefined; disposed = false; } return res; }); return disposePromise; }, }; liveOwners.add(owner); // When the root exits on its own (not via dispose), reconcile ownership by // the *group*. After a short drain window: if the group is empty, deregister; // if descendants are still alive, reap the owned group (no child outlives its // owner). Either way the owner never lingers holding a stale pgid that the OS // could later recycle and a stray dispose could mis-signal. void child.exited .catch(() => undefined) .finally(() => { if (disposed) return; // dispose() owns deregistration if (pgid === undefined) { deregister(); return; } void (async () => { const drained = await pollUntil(() => !groupAlive(pgid), ROOT_EXIT_DRAIN_MS); if (disposed) return; if (drained) { deregister(); return; } // Root exited but the owned group still has descendants: reap them. // dispose() escalates SIGTERM->SIGKILL and deregisters in its finally. await owner.dispose(); })(); }); if (opts.signal) { if (opts.signal.aborted) { void owner.dispose(); } else { onAbort = () => void owner.dispose(); opts.signal.addEventListener("abort", onAbort, { once: true }); } } return owner; } /** Number of currently live owned processes. Exposed for leak assertions/tests. */ export function liveOwnedProcessCount(): number { return liveOwners.size; } /** Dispose every live owned process. For owner-scoped teardown and tests. */ export async function disposeAllOwnedProcesses(): Promise { await Promise.all([...liveOwners].map(owner => owner.dispose().catch(() => undefined))); } // ── F1(b) generic resource owners ──────────────────────────────────────────── type ResourceDisposer = () => void | Promise; const resourceOwners = new Map(); let resourcePostmortemRegistered = false; function ensureResourcePostmortem(): void { if (resourcePostmortemRegistered) return; resourcePostmortemRegistered = true; // Postmortem isolates per-callback failures; swallow the aggregate here so // shutdown continues, while direct callers of disposeAllResourceOwners still // observe the AggregateError. postmortem.register("runtime:resource-owners", () => disposeAllResourceOwners().catch(err => { logger.warn("resource owner postmortem cleanup had failures", { error: err instanceof Error ? err.message : String(err), }); }), ); } /** * Register a non-process resource for postmortem/fatal-exit cleanup. * * Idempotent by `name`: re-registering the same name replaces the prior * disposer (last wins). Returns an unregister function that removes the owner * only while it is still the active registration for that name. */ export function registerResourceOwner(name: string, disposer: ResourceDisposer): () => void { resourceOwners.set(name, disposer); ensureResourcePostmortem(); let unregistered = false; return () => { if (unregistered) return; unregistered = true; if (resourceOwners.get(name) === disposer) { resourceOwners.delete(name); } }; } /** Number of registered resource owners. Exposed for leak assertions/tests. */ export function resourceOwnerCount(): number { return resourceOwners.size; } /** * Run and clear every registered resource disposer. Attempts all disposers even * if some throw, then surfaces the failures as an `AggregateError` so callers * can distinguish "all closed" from "a resource may still be alive". */ export async function disposeAllResourceOwners(): Promise { const disposers = [...resourceOwners.values()]; resourceOwners.clear(); const errors: unknown[] = []; for (const disposer of disposers) { try { await disposer(); } catch (err) { errors.push(err); } } if (errors.length > 0) { throw new AggregateError(errors, `${errors.length} resource disposer(s) failed during teardown`); } }