import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import type { ManagedAgentReport } from "../relay"; import { classifyPidLiveness, isPidAlive, isValidPid, parseProcStateIsZombie, readProcessStartMs, type PidLiveness } from "agent-relay-sdk/process-utils"; import { renderTmuxSessionTarget, tmuxCommand, tmuxHasSession } from "agent-relay-sdk/tmux-utils"; import { LOG_DIR, RUNNER_INFO_DIR, SESSION_DIR, STATE_FILE, TOMBSTONE_DIR } from "./constants"; import { systemdMainPid, systemdUnitDiagnostics, systemdUnitLivenessFromDiagnostics, type SystemdUnitDiagnostics, type SystemdUnitLiveness } from "./systemd"; import { sanitizeFsName } from "agent-relay-sdk/fs-name"; import type { RunnerInfo, SessionRecord, SessionSupervisor } from "./types"; export function logFilePath(name: string): string { return join(LOG_DIR, `${name}.log`); } export function runnerInfoPath(name: string): string { const safe = sanitizeFsName(name, { replacement: "-", trimEdge: true, fallback: "runner" }); return join(RUNNER_INFO_DIR, `${safe}.json`); } // The filesystem key a tombstone/runner-info file for `name` is stored under. tombstonePath // derives the filename from it; the read-side identity binding (#1514 Defect 4) compares a // tombstone's INTERNAL name against this same key so an internal-name≠filename tombstone is // rejected rather than trusted (which would clear/authorize the wrong path). function fsNameKey(name: string): string { return sanitizeFsName(name, { replacement: "-", trimEdge: true, fallback: "runner" }); } export function tombstonePath(name: string): string { return join(TOMBSTONE_DIR, `${fsNameKey(name)}.json`); } export function ensureLogDir(): void { mkdirSync(LOG_DIR, { recursive: true }); } export function ensureSessionDir(): void { mkdirSync(SESSION_DIR, { recursive: true, mode: 0o700 }); } export function ensureRunnerInfoDir(): void { mkdirSync(RUNNER_INFO_DIR, { recursive: true, mode: 0o700 }); } export function ensureTombstoneDir(): void { mkdirSync(TOMBSTONE_DIR, { recursive: true, mode: 0o700 }); } export function saveState(records: SessionRecord[]): void { mkdirSync(join(homedir(), ".agent-relay"), { recursive: true }); // Atomic write: a crash mid-write would otherwise leave truncated JSON and // loadState would silently return [], losing every tracked session. const tmp = `${STATE_FILE}.tmp`; writeFileSync(tmp, JSON.stringify(records, null, 2) + "\n"); renameSync(tmp, STATE_FILE); } export type StateLoadStatus = "loaded" | "missing" | "corrupt"; export interface StateLoadOutcome { /** * "loaded" — the file was read and parsed into a records array (possibly empty). * "missing" — the file does not exist / could not be read. * "corrupt" — the file exists but is not parseable JSON, or not an array. */ status: StateLoadStatus; records: SessionRecord[]; } // A record is structurally valid only when it is a plain object carrying a non-empty // string `name` — the tracked-session identity the whole orchestrator (and the // wedged-session reaper) keys on. #1514 — a logically-corrupt array like `[{}]` or // `[{"name":17}]` parses as a JSON array but yields NO usable name, so the derived // tracked-name set is empty, which would expose every owned idle session to reaping // (fail-open). Any such record makes the file a LOAD FAILURE ("corrupt"), not a valid // empty set, so the reaper skips / rebuilds from reality instead of mass-reaping. // Kept to `name` (not pid/etc.) so it never rejects a genuine record — every spawn // writes a string name (spawn-agent.ts) — while still catching the fail-open shapes. export function isValidSessionRecordShape(value: unknown): value is SessionRecord { if (!value || typeof value !== "object" || Array.isArray(value)) return false; const record = value as Record; // #1514 — trim before the emptiness check so a whitespace-only name (" ", "\n") // is rejected as structurally invalid: it is not a usable tracked-session identity, // and letting it through would seed the tracked-name set with an unmatchable ghost. return typeof record.name === "string" && record.name.trim().length > 0; } // #1514 — the RAW load outcome, so callers can distinguish "genuinely no managed // sessions" (status "loaded", records []) from "managed state was lost/unloaded" // (status "missing"/"corrupt"). The wedged-session reaper MUST skip its cycle on // the latter, and recovery MUST NOT overwrite a missing/corrupt file with a valid // empty array — either would erase the signal and let a state-file loss look like // "every live session is unmanaged", mass-reaping healthy agents. A structurally // invalid array (any record failing isValidSessionRecordShape) is "corrupt", so a // logically-corrupt file can never be mistaken for a positively-loaded empty set. export function loadStateOutcome(stateFile: string = STATE_FILE): StateLoadOutcome { let raw: string; try { raw = readFileSync(stateFile, "utf8"); } catch { return { status: "missing", records: [] }; } try { const parsed = JSON.parse(raw); if (!Array.isArray(parsed)) return { status: "corrupt", records: [] }; if (!parsed.every(isValidSessionRecordShape)) return { status: "corrupt", records: [] }; return { status: "loaded", records: parsed as SessionRecord[] }; } catch { return { status: "corrupt", records: [] }; } } export function loadState(): SessionRecord[] { return loadStateOutcome().records; } export function addSessionRecord(record: SessionRecord): void { const records = loadState().filter((r) => r.name !== record.name); records.push(record); saveState(records); } export function removeSessionRecord(name: string): void { saveState(loadState().filter((r) => r.name !== name)); } // Zombie-aware liveness primitives are shared with the runner via the SDK. // Re-exported so existing `./spawn` consumers (and tests) keep resolving them. export { classifyPidLiveness, isPidAlive, isValidPid, parseProcStateIsZombie }; export type { PidLiveness }; export function sessionSupervisor(record?: Pick): SessionSupervisor { return record?.supervisor ?? { type: "process" }; } export interface SessionRecordLiveness { liveness: SystemdUnitLiveness; // Present only for a systemd-supervised record — the exact diagnostics read that // produced `liveness`, captured at the moment liveness was first checked. Callers // that need to report *why* a session died should reuse this rather than querying // `systemctl show` again later: by then systemd may have already garbage-collected // a `--collect`ed transient unit, and a second query would read back defaulted // (LoadState=not-found) properties instead of the real exit status (#1317). systemd?: SystemdUnitDiagnostics; } export function isSessionRecordAlive(record: SessionRecord): boolean { return sessionRecordLiveness(record) === "alive"; } export function sessionRecordLivenessDetailed(record: SessionRecord): SessionRecordLiveness { const supervisor = sessionSupervisor(record); if (supervisor.type === "systemd" && supervisor.unit) { const diagnostics = systemdUnitDiagnostics(supervisor.unit); return { liveness: systemdUnitLivenessFromDiagnostics(diagnostics, isPidAlive, record.pid), systemd: diagnostics }; } return { liveness: isPidAlive(record.pid) ? "alive" : "dead" }; } export function sessionRecordLiveness(record: SessionRecord): SystemdUnitLiveness { return sessionRecordLivenessDetailed(record).liveness; } export function currentSessionPid(record: SessionRecord): number { const supervisor = sessionSupervisor(record); if (supervisor.type === "systemd" && supervisor.unit) { const pid = systemdMainPid(supervisor.unit); if (pid > 0) return pid; } return record.pid; } export function sessionReportFields(record: Pick): Pick { const supervisor = sessionSupervisor(record); const terminalAvailable = tmuxHasSession(record.name, readRunnerInfo(record)?.tmuxSocket); return { sessionName: record.name, tmuxSession: record.name, supervisor: supervisor.type, ...(supervisor.type === "systemd" && supervisor.unit ? { systemdUnit: supervisor.unit } : {}), terminalSession: record.name, terminalAvailable, }; } export function selectSessionRecord(records: SessionRecord[], input: { agentId?: string; policyName?: string; spawnRequestId?: string; tmuxSession?: string }): SessionRecord | undefined { // #1746 RC-1 — try EVERY provided key, falling through on a MISS rather than early-returning // `undefined` on the first-specified one. A session-name lookup that misses (e.g. the runner's // rendered `.`/`:`→`_` name vs the record's requested name, #1583) must still resolve by // spawnRequestId/agentId/policyName so the host-side shutdown fallback can find the live session. if (input.tmuxSession) { const rendered = renderTmuxSessionTarget(input.tmuxSession); const bySession = records.find((record) => record.name === input.tmuxSession || renderTmuxSessionTarget(record.name) === rendered); if (bySession) return bySession; } // #1746 fwd BLOCKER — each key falls through on a MISS instead of early-returning `undefined`. The // prior code `return`ed the `.find()` result for the first-specified key even when it was undefined, // so a spawnRequestId that missed (or whose policyName conjunct rejected the match) stranded the whole // lookup and never tried agentId/policyName — the exact surviving-agent path the kill-switch must // resolve. This mirrors control.ts `managedAgentShutdownTarget`: match each UNIQUE key on its own // (agentId → spawnRequestId → policyName, per e12b4c2a's stated canonical order) with no cross-key // conjunct, so a divergent policyName can never veto an otherwise-exact agentId/spawnRequestId hit. if (input.agentId) { const byAgentId = records.find((record) => record.agentId === input.agentId); if (byAgentId) return byAgentId; } if (input.spawnRequestId) { const bySpawnRequestId = records.find((record) => record.spawnRequestId === input.spawnRequestId); if (bySpawnRequestId) return bySpawnRequestId; } if (input.policyName) { const policyName = input.policyName; return records .filter((record) => record.policyName === policyName) .reduce((latest, record) => ( !latest || record.startedAt > latest.startedAt ? record : latest ), undefined); } return undefined; } export function findSessionRecord(input: { agentId?: string; policyName?: string; spawnRequestId?: string; tmuxSession?: string }): SessionRecord | undefined { return selectSessionRecord(loadState(), input); } // #1514 — the minimal identity of the runner that owns a managed tmux session, // read from the runner-info file keyed by session name. The runner stamps its own // `process.pid` (plus agentId/provider) there and the file persists across an // orchestrator restart, so it is the ground-truth "who owns this session, and is it // still alive?" signal — independent of the (loss-prone) orchestrator state file. export interface OwnerRunnerInfo { pid: number; agentId?: string; provider?: string; runnerId?: string; startedAt?: number; } export function readOwnerRunnerInfo(name: string): OwnerRunnerInfo | null { try { const parsed = JSON.parse(readFileSync(runnerInfoPath(name), "utf8")); if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; const info = parsed as Record; const pid = info.pid; // #1514 (Defect 1) — the pid must be a semantically-valid OS pid (positive INTEGER in // range) BEFORE it is ever handed to the liveness probe. A corrupt-but-parseable pid // (fractional 1.5, out-of-range 2147483648, 0/negative) is INVALID runner-info: reject // it so the classifier reads null ⇒ "unknown" ⇒ SPARE, rather than letting `process.kill` // throw and collapse to "dead" ⇒ reaping a LIVE session. pid 0 (own process group) is // likewise rejected by isValidPid. if (typeof pid !== "number" || !isValidPid(pid)) return null; return { pid, agentId: typeof info.agentId === "string" ? info.agentId : undefined, provider: typeof info.provider === "string" ? info.provider : undefined, runnerId: typeof info.runnerId === "string" ? info.runnerId : undefined, startedAt: typeof info.startedAt === "number" ? info.startedAt : undefined, }; } catch { return null; } } // #1514 (Defect 3) — a DURABLE proof-of-death record. `cleanupSessionRecord` DELETES the // runner-info file when a runner is confirmed dead, but its owned tmux session can still // linger untracked; with the runner-info gone, the reaper's classifier can no longer see // "dead" and would SPARE the orphan forever. A tombstone (written atomically at cleanup, // keyed by session name, in a SEPARATE dir cleanup never touches) preserves the positive // death evidence so the reaper can reap the lingering session. Cleared once the session is // actually reaped or confirmed gone, and on any name-reuse spawn, so it never over-reaps. export interface RunnerTombstone { name: string; pid: number; startedAt?: number; runnerId?: string; confirmedDeadAt: number; // #1514 (r7) — the OBSERVABLE identity of the tmux session the dead runner left behind, // captured from the live session at the exact moment death was confirmed: its pane pid and // that pane process's OS start time. This is what BINDS the tombstone to one specific // session incarnation. A tombstone can authorize a reap only for a session whose currently // observed pane matches this binding (see tombstoneAuthorizesDeath) — so a tombstone that // never observed the session (stale, forged, copied, or written while no session was // visible) can never kill a session that exists now. Absent when no live session was // observable at confirm time — such a tombstone never authorizes reaping a live session. panePid?: number; paneStartMs?: number; // A required literal marker: a partial/corrupt file missing it is IGNORED on read // (⇒ spare), so a half-written tombstone can never be mistaken for proof of death. marker: "confirmed-dead"; } // Write the tombstone atomically (temp-file + rename) so a crash mid-write can never leave // a truncated file that reads back as partial. Best-effort: a write failure must never break // the cleanup path (worst case: no durable death proof ⇒ the orphan is spared, never reaped). export function writeRunnerTombstone(input: { name: string; pid: number; startedAt?: number; runnerId?: string; confirmedDeadAt: number; panePid?: number; paneStartMs?: number }): void { try { ensureTombstoneDir(); const tombstone: RunnerTombstone = { name: input.name, pid: input.pid, ...(typeof input.startedAt === "number" ? { startedAt: input.startedAt } : {}), ...(input.runnerId ? { runnerId: input.runnerId } : {}), confirmedDeadAt: input.confirmedDeadAt, // The observable session binding (r7): stamp only well-formed values, so a bad capture // degrades to an UNBOUND tombstone (which can never authorize reaping a live session). ...(typeof input.panePid === "number" && isValidPid(input.panePid) ? { panePid: input.panePid } : {}), ...(typeof input.paneStartMs === "number" && Number.isFinite(input.paneStartMs) ? { paneStartMs: input.paneStartMs } : {}), marker: "confirmed-dead", }; const path = tombstonePath(input.name); const tmp = `${path}.tmp`; writeFileSync(tmp, JSON.stringify(tombstone) + "\n", { mode: 0o600 }); renameSync(tmp, path); } catch { // Durable death-proof is a best-effort optimisation of the fail-safe default (spare); // never let its absence become a thrown error in the cleanup path. } } // Read + VALIDATE a tombstone. Any corruption — unreadable, unparseable, wrong shape, a pid // that is not a valid OS pid, a missing/incorrect `confirmed-dead` marker, or an INTERNAL // name that does not map to the file it was read from (#1514 Defect 4) — returns null, so a // partial/tampered/misplaced tombstone is IGNORED (⇒ spare), never mistaken for proof of death. export function readRunnerTombstone(name: string): RunnerTombstone | null { try { const parsed = JSON.parse(readFileSync(tombstonePath(name), "utf8")); if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; const t = parsed as Record; if (t.marker !== "confirmed-dead") return null; if (typeof t.name !== "string" || t.name.trim().length === 0) return null; // #1514 (Defect 4) — filename==name binding: the tombstone's internal name must sanitize // to the SAME key the file is stored under. A tombstone whose internal name ≠ filename is // misplaced/forged — trusting it (or clearing tombstonePath(internalName)) would touch the // WRONG path — so treat it as invalid ⇒ ignored ⇒ spare. if (fsNameKey(t.name) !== fsNameKey(name)) return null; if (typeof t.pid !== "number" || !isValidPid(t.pid)) return null; if (typeof t.confirmedDeadAt !== "number" || !Number.isFinite(t.confirmedDeadAt)) return null; // #1514 (r7) — the observable session binding is optional, but when present it must be // well-formed; a malformed binding makes the whole tombstone invalid (⇒ ignored ⇒ spare) // rather than silently downgrading to an unbound one. if (t.panePid !== undefined && (typeof t.panePid !== "number" || !isValidPid(t.panePid))) return null; if (t.paneStartMs !== undefined && (typeof t.paneStartMs !== "number" || !Number.isFinite(t.paneStartMs))) return null; return { name: t.name, pid: t.pid, startedAt: typeof t.startedAt === "number" ? t.startedAt : undefined, runnerId: typeof t.runnerId === "string" ? t.runnerId : undefined, confirmedDeadAt: t.confirmedDeadAt, panePid: t.panePid as number | undefined, paneStartMs: t.paneStartMs as number | undefined, marker: "confirmed-dead", }; } catch { return null; } } export function clearRunnerTombstone(name: string): void { try { rmSync(tombstonePath(name), { force: true }); } catch {} } /** True iff is a valid, present confirmed-dead marker — durable proof of death. */ export function isConfirmedDeadTombstone(tombstone: RunnerTombstone | null): tombstone is RunnerTombstone { return tombstone !== null && tombstone.marker === "confirmed-dead"; } // #1514 — how much LATER than the recorded runner's `startedAt` a PID's current process // may have started before we call it a DIFFERENT process (PID reuse). `startedAt` is // stamped by the runner at its own launch (runner/src/index.ts), so the runner process's // real OS start time is at/just-before it; a comfortably wide margin absorbs launch // latency and clock/tick rounding, and being wrong here can never cause a reap (a // reused/ambiguous PID is spared either way — see classifyOwnerRunnerLiveness). const RUNNER_PID_REUSE_MARGIN_MS = 60_000; export type OwnerRunnerLiveness = // POSITIVE proof the recorded runner is gone: its PID is entirely absent from the // process table (or is a zombie). The ONLY classification that permits a reap. | "dead" // The recorded PID is present AND identity-consistent with the runner that recorded // it (start time not provably newer than `startedAt`, or unbindable). Healthy → spare. | "alive" // We cannot positively prove death: no/invalid runner-info, OR the PID is present but // bound to a provably-different (reused) process, OR identity is otherwise ambiguous. // Fail closed → spare. Distinguished from "alive" only so reconstruction can avoid // enriching an adopted record with a PID that is now an unrelated process. | "unknown"; /** * #1514 — classify the liveness of the runner that owns a managed session, from its * best-effort runner-info file. This is the SINGLE fail-closed decision the reaper and * state-reconstruction both consult, so "absence of metadata ⇒ spare" is structurally * guaranteed in one place: * - no/unreadable/invalid runner-info ⇒ "unknown" (runner-info is written non-atomically * and best-effort, so its absence is NOT evidence the runner is gone). * - PID probe returns "dead" (ESRCH/absent, or a confirmed zombie) ⇒ "dead" (the recorded * runner positively terminated). This is the ONLY branch that permits a reap. * - PID probe returns "unknown" (EPERM, an invalid pid, or ANY other error — Defect 2) ⇒ * "unknown". A present-but-unprobeable runner is NOT proof of death ⇒ fail closed ⇒ spare. * - PID present but its process started well after the recorded `startedAt` ⇒ "unknown" * (the PID was reused by a different process — we bind PID→identity rather than trust a * bare numeric PID, but per the fail-closed mandate an ambiguous/reused PID is SPARED). * - PID present and identity-consistent (or unbindable) ⇒ "alive". * The `classify` dep is a TRI-STATE probe (classifyPidLiveness), NOT two-state isPidAlive: * that is what lets EPERM/ambiguity resolve to "unknown" (spare) instead of "dead" (reap). */ export function classifyOwnerRunnerLiveness( info: OwnerRunnerInfo | null, deps: { classify?: (pid: number) => PidLiveness; startMs?: (pid: number) => number | null } = {}, ): OwnerRunnerLiveness { if (!info) return "unknown"; // fail closed — missing/unreadable/invalid metadata proves nothing const classify = deps.classify ?? classifyPidLiveness; const liveness = classify(info.pid); if (liveness === "unknown") return "unknown"; // EPERM / unprobeable / ambiguous ⇒ spare (Defect 2) if (liveness === "dead") return "dead"; // positive proof: the recorded PID is absent or a zombie // The PID is alive. Bind it to the recorded runner identity to rule out PID reuse. if (typeof info.startedAt === "number") { const startMs = (deps.startMs ?? readProcessStartMs)(info.pid); if (startMs !== null && startMs > info.startedAt + RUNNER_PID_REUSE_MARGIN_MS) { // The live process at this PID started well after the runner recorded its own // launch — it cannot be that runner. Ambiguous re-use ⇒ fail closed ⇒ spare. return "unknown"; } } return "alive"; } // #1514 (r6) — injectable probes for the single owner-death gate, so every "is the owning // runner dead?" decision can be exercised deterministically in tests without real /proc. export interface OwnerDeathProbeDeps { classify?: (pid: number) => PidLiveness; startMs?: (pid: number) => number | null; readInfo?: (name: string) => OwnerRunnerInfo | null; readTombstone?: (name: string) => RunnerTombstone | null; } // #1514 (r7) — the OBSERVABLE identity of a live tmux session's pane process: the pane pid // tmux reports for the session, plus that process's OS start time (when readable). This is // the real-time signal the tombstone path binds to — unlike runner-info (best-effort, can be // absent for a live session) it is read from the session itself. export interface SessionPaneBinding { panePid: number; paneStartMs?: number; } /** * Capture the observable pane binding of a live tmux session, best-effort. Returns null when * the session (or its pane pid) is not observable — callers must treat null as "no observable * session", which for the tombstone WRITER means the tombstone is left UNBOUND (and an unbound * tombstone can never authorize reaping a live session — fail closed). */ export function captureSessionPaneBinding( name: string, tmuxSocket?: string, deps: { startMs?: (pid: number) => number | null } = {}, ): SessionPaneBinding | null { try { const result = Bun.spawnSync(tmuxCommand(tmuxSocket, "list-panes", "-s", "-t", name, "-F", "#{pane_pid}"), { stdin: "ignore", stdout: "pipe", stderr: "ignore", }); if (result.exitCode !== 0) return null; for (const line of result.stdout.toString().split("\n")) { const pid = Number(line.trim()); if (!Number.isFinite(pid) || !isValidPid(pid)) continue; const startMs = (deps.startMs ?? readProcessStartMs)(pid); return { panePid: pid, ...(startMs !== null ? { paneStartMs: startMs } : {}) }; } return null; } catch { return null; } } // #1514 (r7, closes the r6 CRITICAL) — decide whether a DURABLE tombstone authorizes "owner // dead" for the OBSERVED session. FAIL CLOSED + IDENTITY BOUND, twice over: a tombstone counts // as proof of death ONLY when ALL of the following hold — // 1. it is a valid, present confirmed-dead marker (readRunnerTombstone already rejects // corrupt/partial/marker-less/invalid-pid AND internal-name≠filename tombstones), and // 2. its internal name EXACTLY equals the session name — sanitize-key equality is a filename // binding, not an identity; a name that merely sanitizes to the same key is rejected, and // 3. when the current runner-info is present (an ambiguous live-ish incarnation), the // tombstone belongs to the SAME incarnation — (pid, startedAt, runnerId) all match — so a // stale/forged tombstone from a prior incarnation can never override present runner-info, and // 4. the tombstone's recorded runner pid is POSITIVELY dead RIGHT NOW (absent/zombie), // pid-reuse-bound via classifyOwnerRunnerLiveness, and // 5. the tombstone is BOUND to the observable CURRENT session (the r6 CRITICAL closure): it // recorded the session's pane identity at the moment death was confirmed, and that binding // matches the pane observed NOW — // - the caller observed a valid pane pid for the live session (no observable signal ⇒ spare), // - the tombstone's recorded panePid equals it (different/unknown incarnation ⇒ spare), // - a pane that probes "unknown" (EPERM/unprobeable) ⇒ spare, // - a pane that probes ALIVE must be the SAME process the tombstone observed — its OS // start time must exactly equal the recorded paneStartMs (pane-pid reuse guard; an // unreadable/unrecorded start time cannot bind ⇒ spare). This is the genuine-wedge // case: the dead runner's abandoned TUI still running in the pane. // - a pane that probes DEAD (ESRCH/zombie) is positive evidence the bound pane is gone; // when both start times are readable they must still agree (else: different process ⇒ spare). // A tombstone ALONE — however valid — never authorizes death: without a matching observable // session binding every path above returns false (spare). The only writer of a bound tombstone // is cleanupSessionRecord at the moment the single gate confirmed death, so a binding that // matches the session observed now is proof the death confirmation was about THIS incarnation. // Any mismatch / reuse / staleness / ambiguity ⇒ false (spare). function tombstoneAuthorizesDeath( tombstone: RunnerTombstone | null, session: { name: string; panePid?: number }, info: OwnerRunnerInfo | null, deps: Pick, ): boolean { if (!isConfirmedDeadTombstone(tombstone)) return false; if (tombstone.name !== session.name) return false; // EXACT name identity (r6 finding 2) if (info) { // Incarnation-bind against present runner-info: a tombstone from a different incarnation // is stale/forged and must never authorize death for the currently-registered runner. if (tombstone.pid !== info.pid) return false; if ((tombstone.startedAt ?? undefined) !== (info.startedAt ?? undefined)) return false; if ((tombstone.runnerId ?? undefined) !== (info.runnerId ?? undefined)) return false; } // The recorded dead runner must STILL probe positively dead (pid-reuse-bound). if (classifyOwnerRunnerLiveness( { pid: tombstone.pid, startedAt: tombstone.startedAt, runnerId: tombstone.runnerId }, deps, ) !== "dead") return false; // 5. Bind to the observable CURRENT session owner (r7). A tombstone alone never authorizes. const panePid = session.panePid; if (panePid === undefined || !isValidPid(panePid)) return false; // no observable signal ⇒ spare if (tombstone.panePid === undefined || tombstone.panePid !== panePid) return false; // unbound / different incarnation ⇒ spare const classify = deps.classify ?? classifyPidLiveness; const paneLiveness = classify(panePid); if (paneLiveness === "unknown") return false; // unprobeable pane ⇒ spare const currentPaneStartMs = (deps.startMs ?? readProcessStartMs)(panePid); if (paneLiveness === "alive") { // Live pane: require exact start-time identity with the pane the tombstone observed. if (tombstone.paneStartMs === undefined || currentPaneStartMs === null) return false; if (currentPaneStartMs !== tombstone.paneStartMs) return false; } else if (currentPaneStartMs !== null && tombstone.paneStartMs !== undefined && currentPaneStartMs !== tombstone.paneStartMs) { // Dead-but-still-readable (zombie) pane that is provably a DIFFERENT process ⇒ spare. return false; } return true; } // #1514 (r6/r7) — THE SINGLE fail-closed, identity-bound gate. This is the ONLY predicate that // authorizes (a) writing a confirmed-dead tombstone + deleting runner-info (cleanupSessionRecord), // (b) reaping a managed session (wedged-session reaper, via isManagedSessionOrphaned), and // (c) leaving a session unadopted at state reconstruction (reconstructOwnedLiveSessions). It // returns true ONLY on positive, identity-bound proof the owning runner terminated: // - runner-info present + its recorded pid probes "dead" (ESRCH/zombie) ⇒ dead. The ONLY // branch that concludes death from live metadata. // - runner-info present + pid "alive" (identity-consistent) ⇒ NOT dead — a live runner is // never an orphan, and any stale tombstone is ignored while it lives. // - runner-info absent/ambiguous ("unknown": missing/unreadable/invalid, EPERM, reused pid, // systemd-unavailable) ⇒ dead ONLY IF a durable tombstone proves it — and (r7) a tombstone // proves it only when it is exactly-name-matched, identity-matched against any present // runner-info, its recorded runner pid probes positively dead NOW, AND it is bound to the // observable CURRENT session via the caller-observed pane pid (`session.panePid`, from // tmux `#{pane_pid}`) matching the pane identity the tombstone captured at confirm time. // A caller that cannot observe the session's pane passes no panePid, and the tombstone // path then NEVER concludes death. Otherwise NOT dead. // EPERM / any probe error / invalid pid / missing runner-info / stale-or-mismatched tombstone / // pid reuse / unbound tombstone / unobservable pane / any ambiguity ⇒ false (spare). Leaking a // wedged session is acceptable; killing a live one is not — so this predicate never concludes // "dead" on anything short of proof. export function isOwnerConfirmedDead(session: { name: string; panePid?: number }, deps: OwnerDeathProbeDeps = {}): boolean { const readInfo = deps.readInfo ?? readOwnerRunnerInfo; const readTombstone = deps.readTombstone ?? readRunnerTombstone; const info = readInfo(session.name); const liveness = classifyOwnerRunnerLiveness(info, { classify: deps.classify, startMs: deps.startMs }); if (liveness === "alive") return false; // a live runner is never dead — ignore any stale tombstone if (liveness === "dead") return true; // positive proof via present runner-info // liveness === "unknown": runner-info absent/ambiguous. Only a durable, identity-bound, // positively-dead, OBSERVED-SESSION-BOUND tombstone may conclude death here (Defect 3 / r7). return tombstoneAuthorizesDeath(readTombstone(session.name), session, info, { classify: deps.classify, startMs: deps.startMs }); } // #1514 — the POSITIVE-ORPHAN test used by the wedged-session reaper. A thin adapter over the // single gate (isOwnerConfirmedDead) so the reaper and the tombstone WRITER share ONE // fail-closed, identity-bound decision — there is no second code path that can classify a live // runner as dead. `session.panePid` is the live session's observed pane pid (the reaper already // captures tmux `#{pane_pid}`); without it the tombstone path can never conclude death. Returns // true (reap-eligible orphan) ONLY on positive proof the owning runner terminated; every // ambiguous/absent/stale/unbound case FAILS CLOSED and returns false (spare). export function isManagedSessionOrphaned( session: { name: string; panePid?: number }, deps: OwnerDeathProbeDeps = {}, ): boolean { return isOwnerConfirmedDead(session, deps); } export function readRunnerInfo(record: Pick): RunnerInfo | null { if (!record.runnerInfoFile) return null; try { const parsed = JSON.parse(readFileSync(record.runnerInfoFile, "utf8")); if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; const info = parsed as Record; if (typeof info.controlUrl !== "string" || !info.controlUrl.startsWith("http://127.0.0.1:")) return null; return { agentId: typeof info.agentId === "string" ? info.agentId : record.agentId, runnerId: typeof info.runnerId === "string" ? info.runnerId : "", provider: typeof info.provider === "string" ? info.provider : record.provider, controlUrl: info.controlUrl, tmuxSession: typeof info.tmuxSession === "string" ? info.tmuxSession : undefined, tmuxSocket: typeof info.tmuxSocket === "string" ? info.tmuxSocket : undefined, pid: typeof info.pid === "number" ? info.pid : undefined, startedAt: typeof info.startedAt === "number" ? info.startedAt : undefined, registeredAt: typeof info.registeredAt === "number" ? info.registeredAt : undefined, }; } catch { return null; } }