import { join } from "node:path"; import { chmodSync, closeSync, fsyncSync, lstatSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; import { randomUUID } from "node:crypto"; import { homedir } from "node:os"; import { shellQuote } from "agent-relay-sdk/shell-utils"; /** * #1509 — the self-upgrade restart guard's IDENTITY, SCRIPT, and REAPER. * * This module owns everything about the launchd restart guard that is orthogonal to planning/ * dispatching an upgrade (which lives in ./self-upgrade.ts): the per-generation identity scheme * (§2), the guard shell script (§8), the one-shot plist (§13), and the running-state reaper (§6, * §7, §13). See docs/self-upgrade-guard-design.md for the full concurrency argument. * * r7 (§13): on launchd the guard is a ONE-SHOT `launchctl bootstrap` job (RunAtLoad, KeepAlive=false), * NOT a KeepAlive `launchctl submit` job. It carries NO status/completion marker — its presence-and- * running state in the launchd listing IS the in-progress signal, so a not-running guard is terminal * (done or dead, both safe) and a dead guard can never wedge dispatch. The r5 marker machinery below * survives only to classify PRE-r7 KeepAlive guards during the one-time upgrade transition. * * r12+r13 (§18): every "provably absent / provably foreign / provably empty" determination in this * module is TOTAL over its input domain and fails CLOSED on ambiguity, through exactly ONE audited * implementation per determination: marker presence = defaultMarkerPresent (lstat — only a clean * ENOENT is absence); launchd listing trust = readLaunchctlGuardSnapshot (exit 0 AND well-formed * shape AND our own live job present as the completeness anchor — r13); systemd listing trust = * parseSystemdGuardUnits (well-formed rows; an unknown ActiveState contends); ExecStart ownership = * the couldBeOurGuard / isProvablyOurGuard predicate PAIR with opposite fail-defaults (r13 — block * on any could-be, neutralize only on strict positive proof); numeric manager readbacks fully * anchored (parsePsEtime, parseMainPidValue); the crash-loop store counts DISTINCT pids (r13). */ // ERE matched against /api/health by the readiness probe inside the restart guard. export const HEALTH_PATTERN = '"status"[[:space:]]*:[[:space:]]*"ok"'; // ───────────────────────────────────────────────────────────────────────────── // #1509 r8 (Finding 3) — structural bounds so a hung external command can never leave the one-shot // guard running/blocking. Every launchctl/systemctl invocation inside the guard is wrapped in a // portable `bounded` timeout (no `timeout` binary on macOS), curl carries --connect-timeout/--max-time, // and the verify loop is capped by a wall-clock DEADLINE — so the guard's TOTAL runtime is bounded by // construction (see the ~worst-case sum in the design doc §13/§14), not merely by attempt count. // ───────────────────────────────────────────────────────────────────────────── /** Hard per-command timeout (seconds) for each launchctl/systemctl call INSIDE the guard script. */ export const GUARD_CMD_TIMEOUT_SECONDS = 10; /** curl health-probe cap (seconds) — bounds every readiness probe in the verify loop. */ export const GUARD_CURL_MAX_SECONDS = 5; /** curl connect timeout (seconds). */ export const GUARD_CURL_CONNECT_SECONDS = 3; /** * Wall-clock DEADLINE (seconds) on the guard's restart+verify loop — the REAL bound (the pre-r8 loop * was ~30 × (5s curl + 2s sleep) ≈ 210s, NOT the ~60s the doc claimed). The loop now breaks once this * many seconds have elapsed regardless of how slow any single curl was. */ export const GUARD_VERIFY_DEADLINE_SECONDS = 60; /** * Hard cap (seconds) on the WHOLE guard unit's wall-clock runtime. Passed to systemd as * `RuntimeMaxSec` so systemd itself SIGTERM/SIGKILLs a wedged guard transient unit even if every * in-script bound somehow failed. (No launchd equivalent exists, but the in-script per-command * `bounded` timeouts + loop deadline already bound the launchd guard by construction.) */ export const GUARD_MAX_RUNTIME_SECONDS = 120; /** Timeout (ms) the ORCHESTRATOR applies to the guard-dispatch command (launchctl bootstrap / systemd-run). */ export const GUARD_DISPATCH_TIMEOUT_MS = 15_000; /** * Timeout (ms) on every ORCHESTRATOR-side manager call (`launchctl list/remove`, `systemctl * list-units/show/stop/reset-failed`, `ps`) — #1509 r10 Finding 5. These calls are synchronous by * design (the precondition→swap sequence must have no await), so a hung manager binary would * otherwise wedge the command poller's event loop indefinitely. Bounded ⇒ worst case a few seconds * of stall, and the call reports failure (fail closed) instead of hanging. */ export const MANAGER_CMD_TIMEOUT_MS = 10_000; /** * #1509 r10 (Finding 2) — the GENEROUS wall-clock bound (seconds) past which a RUNNING own guard is * judged hung and neutralized (launchd: `launchctl remove `; systemd: `systemctl stop` * + `reset-failed `) instead of blocking dispatch forever. A legitimate guard finishes in * ≤ ~120s by construction (GUARD_MAX_RUNTIME_SECONDS; even the unbounded-ish r7/r8 generations were * nominally ≤ ~210s), so 15 minutes is ≥ 4× any legitimate restart+verify. Ambiguity NEVER reaps: * when the elapsed runtime cannot be read, the guard is treated as legitimately working (block). */ export const HUNG_GUARD_NEUTRALIZE_SECONDS = 900; /** * Synchronous, HARD-BOUNDED external command (#1509 r10 Finding 5) — the orchestrator-side analogue * of the guard script's `bounded` helper. Every default manager adapter below routes through this, * so a hung `launchctl`/`systemctl`/`ps` can stall the event loop for at most `timeoutMs` (SIGKILL — * a manager stuck in the kernel may ignore SIGTERM) and then reports exit 1, which every caller * already treats fail-closed. Synchronous on purpose: the dispatch precondition must run with no * await between its final snapshot and the symlink swap. */ export function spawnSyncBounded(cmd: string[], timeoutMs = MANAGER_CMD_TIMEOUT_MS): { exitCode: number; stdout: string } { try { const r = Bun.spawnSync({ cmd, stdout: "pipe", stderr: "ignore", timeout: timeoutMs, killSignal: "SIGKILL" }); return { exitCode: r.exitCode ?? 1, stdout: new TextDecoder().decode(r.stdout) }; } catch { return { exitCode: 1, stdout: "" }; } } /** * Parse `ps -o etime=` output (`[[dd-]hh:]mm:ss`) into elapsed seconds; null on anything else. * `etime` (not `etimes`) because macOS/BSD ps has no `etimes` keyword — this is the portable form. */ export function parsePsEtime(raw: string): number | null { const m = /^(?:(?:(\d+)-)?(\d+):)?(\d{1,2}):(\d{2})$/.exec(raw.trim()); if (!m) return null; const days = m[1] ? Number(m[1]) : 0; const hours = m[2] ? Number(m[2]) : 0; return ((days * 24 + hours) * 60 + Number(m[3])) * 60 + Number(m[4]); } /** * STRICT `systemctl show -p MainPID --value` parse (#1509 r12 §18): the value must be EXACTLY one * positive integer. r11 used parseInt, which accepts a numeric PREFIX of garble ("123 456", * "123junk" ⇒ 123), so a corrupt exit-0 readback could yield a WRONG pid whose elapsed runtime then * fed the hung-guard bound — a potential false "hung" (and neutralize) on a young own guard. * Anything but a clean positive integer ⇒ null ⇒ elapsed unknown ⇒ block-without-neutralize. */ export function parseMainPidValue(raw: string): number | null { const trimmed = raw.trim(); if (!/^\d+$/.test(trimmed)) return null; const pid = Number(trimmed); return Number.isSafeInteger(pid) && pid > 0 ? pid : null; } /** Elapsed wall-clock seconds of a live PID via bounded `ps -p -o etime=`; null when unknown. */ function processElapsedSecondsViaPs(pid: number): number | null { if (!Number.isInteger(pid) || pid <= 0) return null; const r = spawnSyncBounded(["ps", "-p", String(pid), "-o", "etime="]); if (r.exitCode !== 0) return null; return parsePsEtime(r.stdout); } /** Resolve the runtime symlink path from an optional detected prefix (shared default otherwise). */ export function resolveRuntimePath(runtimePrefix?: string): string { return runtimePrefix ?? join(homedir(), ".agent-relay", "runtime"); } // ───────────────────────────────────────────────────────────────────────────── // #1509 r5 — generation identity (§2): persisted monotonic sequence + collision-free uuid. // ───────────────────────────────────────────────────────────────────────────── /** * A self-upgrade guard generation (#1509 r5 §2). `genid` = `s${seq}.${uuidhex}` is the single * token stamped into EVERY per-generation path (script/status/result/label) and into the marker * content — so two guards can never collide on any path (uuid) and ordering is total (seq). */ export interface GuardGeneration { /** Monotonic persisted sequence — total ordering, immune to clock rollback + same-ms collision. */ seq: number; /** crypto.randomUUID() with hyphens stripped (32 hex) — collision-free identity. */ uuidhex: string; /** `s${seq}.${uuidhex}`. */ genid: string; } /** The persisted monotonic-sequence file path (`.upgrade-guard-seq`). */ export function upgradeGuardSeqPath(runtimePath: string): string { return `${runtimePath}.upgrade-guard-seq`; } /** * Allocate a fresh guard generation (#1509 r5 §2). MUST be called under the single-flight lock. * * ORDERING FROM A COUNTER, IDENTITY FROM A UUID — never a clock (§12.6). seq is read from the * persisted file (missing/corrupt ⇒ 0, so ordering simply restarts at 1 — §11.2, harmless because * the uuid still makes every generation identity distinct), incremented, and written back * atomically (temp + fsync + rename). uuidhex is a fresh random 32-hex string. Date.now() is NEVER * consulted for either the sequence or the identity. * * FAIL-CLOSED (§2.4): a failure to PERSIST the incremented counter (temp write / fsync / rename) * throws, so the caller aborts the dispatch rather than submitting a guard whose ordering was never * durably recorded. (A read failure is NOT fatal — it is the normal first-upgrade/reset case.) */ export function allocateGuardGeneration(runtimePath: string): GuardGeneration { const seqPath = upgradeGuardSeqPath(runtimePath); const uuidhex = randomUUID().replace(/-/g, ""); let n = 0; try { const parsed = parseInt(readFileSync(seqPath, "utf8").trim(), 10); if (Number.isFinite(parsed) && parsed > 0) n = parsed; } catch { n = 0; // missing or corrupt seq file — bootstrap at 0 (§11.2); uuid keeps identity distinct. } const seq = n + 1; // Persist atomically; any throw here fails the whole allocation closed (do NOT dispatch). const tmpPath = `${seqPath}.tmp.${uuidhex}`; const fd = openSync(tmpPath, "w", 0o644); try { writeFileSync(fd, `${seq}\n`); fsyncSync(fd); } finally { closeSync(fd); } renameSync(tmpPath, seqPath); return { seq, uuidhex, genid: `s${seq}.${uuidhex}` }; } /** Atomic file write: temp (fresh uuid) + fsync + rename over the target; chmod if a mode is given. */ export function atomicWriteFile(path: string, content: string, mode?: number): void { const tmpPath = `${path}.tmp.${randomUUID().replace(/-/g, "")}`; const fd = openSync(tmpPath, "w", mode ?? 0o644); try { writeFileSync(fd, content); fsyncSync(fd); } finally { closeSync(fd); } renameSync(tmpPath, path); if (mode !== undefined) chmodSync(path, mode); } // ───────────────────────────────────────────────────────────────────────────── // #1509 r5 — per-generation path + label helpers (§4). // ───────────────────────────────────────────────────────────────────────────── /** The own-unit guard-label BASE (`${selfUnit}.upgrade-guard`) — the own-vs-foreign scoping key. */ export function ownUpgradeGuardBase(selfUnit: string): string { return `${selfUnit}.upgrade-guard`; } /** The exact per-generation launchd label a dispatch submits under (`.`). */ export function guardLabelForGenid(base: string, genid: string): string { return `${base}.${genid}`; } /** The per-generation status-marker path (`.upgrade-guard-status.`), or the * shared r4 marker path (`.upgrade-guard-status`) when no genid is given. */ export function upgradeGuardStatusPathFor(runtimePath: string, genid?: string): string { return genid ? `${runtimePath}.upgrade-guard-status.${genid}` : `${runtimePath}.upgrade-guard-status`; } /** The per-generation guard-script path (`.upgrade-guard..sh`). */ export function upgradeGuardScriptPathFor(runtimePath: string, genid: string): string { return `${runtimePath}.upgrade-guard.${genid}.sh`; } /** The per-generation result-marker path (`.upgrade-guard-result.`). */ export function upgradeGuardResultPathFor(runtimePath: string, genid: string): string { return `${runtimePath}.upgrade-guard-result.${genid}`; } /** The per-generation one-shot launchd plist path (`.upgrade-guard..plist`, #1509 r7 §13). */ export function upgradeGuardPlistPathFor(runtimePath: string, genid: string): string { return `${runtimePath}.upgrade-guard.${genid}.plist`; } /** Minimal XML text escape for the per-generation guard plist (label/path values, #1509 r7 §13). */ function xmlEscape(value: string): string { return value .replace(/&/g, "&") .replace(//g, ">") .replace(/"/g, """) .replace(/'/g, "'"); } /** * Build the per-generation ONE-SHOT launchd plist that runs the restart guard exactly once * (#1509 r7 §13). The critical properties versus the old `launchctl submit` job: * - `RunAtLoad=true` starts the guard immediately at bootstrap. * - `KeepAlive` is FALSE (explicit): once the guard process exits, launchd NEVER relaunches * it. This is the whole point of the r7 redesign — a not-running one-shot job is TERMINAL * (done or dead, indistinguishable and both safe to re-dispatch over), so there is no * best-effort completion marker that can lie and no KeepAlive relaunch to wedge dispatch. * A fixed default PATH is set so the guard's bare-name tools (launchctl/curl/grep/mv/…) resolve * under launchd's otherwise-minimal environment. */ export function buildGuardPlist(label: string, scriptPath: string): string { return [ '', '', '', "", " Label", ` ${xmlEscape(label)}`, " ProgramArguments", " ", " /bin/sh", ` ${xmlEscape(scriptPath)}`, " ", " RunAtLoad", " ", // KeepAlive FALSE: a one-shot that never relaunches — the structural fix for #1509. " KeepAlive", " ", " ProcessType", " Background", " EnvironmentVariables", " ", " PATH", " /usr/bin:/bin:/usr/sbin:/sbin", " ", "", "", "", ].join("\n"); } // ───────────────────────────────────────────────────────────────────────────── // #1509 r5 — the restart guard shell script (§8). // ───────────────────────────────────────────────────────────────────────────── /** * Build the full POSIX-sh restart guard as a self-contained script. It is written to a * file and dispatched as `sh `, which is what makes the shell VALID and survivable: * the prior version string-joined fragments with "; " and emitted `… do; if curl …`, a * hard parse error (sh AND bash, exit 2). `sh -c` parses the whole string before running, * so the guard died at PARSE and the leading restart never executed — the upgrade swapped * the runtime symlink and silently restarted NOTHING while reporting success (#1315). * * The guard now: * - gates on the new runtime's ExecStart target being executable (test -x) and rolls the * symlink back WITHOUT restarting if it is missing (the #1312 203/EXEC landmine); * - proves the restart actually replaced the process — captures MainPID before, requires * a different nonzero MainPID after, because the OLD process serves /api/health fine; * - requires a stable-healthy /api/health in addition to the PID change; * - on success drops the rollback pointer AND the parked-aside original runtime (.legacy); * - on any failure restores the previous runtime (single atomic rename over the live * symlink — #1287 Bug B), restarts onto it, and exits nonzero; * - always records an `ok|rolled-back ` marker (atomic temp+rename — #1509 r5 §5) so * verification can observe the outcome after the orchestrator reconnects, instead of * trusting the dispatch exit code. * - on launchd, runs as a ONE-SHOT bootstrapped job (`RunAtLoad`, `KeepAlive=false` — #1509 * r7 §13), so once it exits launchd NEVER relaunches it. It still BOOTS ITSELF OUT before * every exit path (`launchctl bootout`, with `remove` as a fallback, plus an EXIT trap) to * keep the launchd listing clean — but this is now purely COSMETIC: a one-shot that fails to * self-remove sits loaded-but-idle at PID `-` and can neither loop nor wedge a later dispatch * (a not-running one-shot is terminal). This replaces the old KeepAlive `launchctl submit` * job, whose failure-to-self-remove re-ran the guard forever (~every 10s), SIGTERM-restart- * looping the orchestrator (#1509). No-op on systemd (self-collecting transient unit). * - carries NO status/completion marker (#1509 r7 §13). The old `active|done ` marker was * a best-effort completion signal that could lie (a guard that died before writing `done` left * a forever-`active` marker that wedged all future upgrades). Under the one-shot model the job's * PRESENCE-AND-RUNNING state in the launchd listing IS the in-progress signal, so no marker is * needed and none is written. The `ok|rolled-back` RESULT marker (below) is unrelated — it is * read by the orchestrator to REPORT the outcome, never to gate reaping or dispatch. */ export function buildArtifactRestartGuard( platform: "systemd" | "launchd", unit: string, runtimePath: string, readinessUrl: string, service?: string, guardLabel?: string, guardDomainTarget?: string, resultPath?: string, plistPath?: string, ): string { const previous = `${runtimePath}.previous`; const legacy = `${runtimePath}.legacy`; // systemd keeps the shared result path; launchd passes a per-generation one (#1509 r5 §4). const result = resultPath ?? `${runtimePath}.upgrade-guard-result`; // The unit's ExecStart / ProgramArguments target, resolved through the NEW runtime // symlink. materializeArtifactBins links it and chmod +x's the source; its absence is // the #1312 203/EXEC crash-loop, so gate on it before we ever tear the service down. const execTarget = join(runtimePath, "node_modules", ".bin", "agent-relay-orchestrator"); const healthUrl = `${readinessUrl}/api/health`; // GNU mv needs -T to replace the symlink itself; BSD mv needs -h. Either way rollback is // a single atomic rename over the live pointer — .previous is always a symlink, so a // prior real directory rolls back without ENOTDIR and never through a no-pointer hole. const mvFlag = platform === "systemd" ? "-fT" : "-fh"; const T = GUARD_CMD_TIMEOUT_SECONDS; // #1509 r8 (Finding 3): every launchctl/systemctl call runs under `bounded` so a hung invocation // cannot leave the guard running (and thus blocking dispatch) forever. const pidFn = platform === "systemd" ? `unit_pid() { bounded ${T} systemctl --user show -p MainPID --value "$UNIT" 2>/dev/null || echo 0; }` : `unit_pid() { bounded ${T} launchctl list "$UNIT" 2>/dev/null | sed -n 's/.*"PID" = \\([0-9][0-9]*\\).*/\\1/p'; }`; const restartFn = platform === "systemd" ? `restart_unit() { bounded ${T} systemctl --user restart "$UNIT"; }` : `restart_unit() { bounded ${T} launchctl kickstart -kp "$SERVICE"; }`; // #1509 r7 §13: on launchd the guard runs as a ONE-SHOT bootstrapped job (KeepAlive=false), so // self-removal is now COSMETIC (list hygiene) rather than loop-critical — but we still do it. // `launchctl bootout` is the counterpart to `bootstrap`; `remove` is a legacy fallback; the // per-generation plist file is deleted too. No-op on systemd (self-collecting transient unit). // #1509 r8: bootout/remove are `bounded` too, so a hung self-remove cannot leave the guard alive. const selfRemoveFn = platform === "launchd" ? `self_remove() { bounded ${T} launchctl bootout ${shellQuote(guardDomainTarget ?? "")} 2>/dev/null || bounded ${T} launchctl remove ${shellQuote(guardLabel ?? "")} 2>/dev/null || true; rm -f ${shellQuote(plistPath ?? "")} 2>/dev/null || true; }` : "self_remove() { :; }"; return [ "#!/bin/sh", `# agent-relay self-upgrade restart guard (${platform}) — generated, do not edit.`, "set -u", `RUNTIME=${shellQuote(runtimePath)}`, `PREVIOUS=${shellQuote(previous)}`, `LEGACY=${shellQuote(legacy)}`, `UNIT=${shellQuote(unit)}`, `SERVICE=${shellQuote(service ?? "")}`, `EXEC_TARGET=${shellQuote(execTarget)}`, `HEALTH_URL=${shellQuote(healthUrl)}`, `HEALTH_PATTERN=${shellQuote(HEALTH_PATTERN)}`, `RESULT=${shellQuote(result)}`, // #1509 r8 (Finding 3): portable per-command hard timeout — no `timeout` binary exists on macOS, // so run the command in the background and a detached watchdog TERM→KILLs it if it overruns. The // command's stdout is preserved (it inherits our fds) so `$(bounded … launchctl list …)` still // captures output; the watchdog is fully redirected so it never pollutes or holds the pipe open. // // #1509 r9 (Finding 4): the bounded command runs in its OWN PROCESS GROUP so a hung CHILD (and its // descendants) is signalled as a group. The r8 `bounded` TERM/KILLed only the immediate PID, so a // surviving child kept a command-substitution pipe open past the bound (repro: `bounded 1 sh -c // "printf hello; sleep 5"` took ~5s, not ~1s), and manager-side jobs a killed systemctl/launchctl // had already forked lived on. We prefer `setsid` (always present on Linux/systemd) to start a new // session+group; on macOS there is no `setsid` binary, but /bin/sh is bash, so we fall back to // monitor mode (`set -m`), which puts each backgrounded job in its own group. Either way `$!` is // the group leader (PGID == PID), so `kill -TERM -$pid` signals the WHOLE tree; the leader PID is // also signalled directly as a belt-and-suspenders if no separate group was created. stdout/exit // code are preserved (the command inherits our fds). "if command -v setsid >/dev/null 2>&1; then _AR_SETSID=1; else _AR_SETSID=0; fi", "bounded() {", " _bt=$1; shift", ' if [ "$_AR_SETSID" = 1 ]; then', ' setsid "$@" &', " else", ' set -m 2>/dev/null; "$@" &', " set +m 2>/dev/null", " fi", " _bp=$!", ' ( sleep "$_bt"; kill -TERM "-$_bp" 2>/dev/null; kill -TERM "$_bp" 2>/dev/null; sleep 1; kill -KILL "-$_bp" 2>/dev/null; kill -KILL "$_bp" 2>/dev/null ) >/dev/null 2>&1 &', " _bw=$!", ' wait "$_bp" 2>/dev/null', " _brc=$?", ' kill "$_bw" 2>/dev/null', ' wait "$_bw" 2>/dev/null', ' return "$_brc"', "}", // Monotonic-ish wall clock for the verify-loop deadline; `echo 0` disables the deadline if date is absent. "now_s() { date +%s 2>/dev/null || echo 0; }", // Result marker written atomically (temp + rename — §5) so a reader never sees a torn line. "mark() { printf '%s %s\\n' \"$1\" \"$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\" > \"$RESULT.tmp.$$\" 2>/dev/null && mv -f \"$RESULT.tmp.$$\" \"$RESULT\" 2>/dev/null || true; }", pidFn, restartFn, selfRemoveFn, // #1509 r7 §13: boot our own one-shot job out on ANY exit (belt to the explicit self_remove // calls below) so the launchd listing stays clean. Cosmetic under KeepAlive=false (a one-shot // that skips this sits idle, never loops). Harmless on systemd. There is deliberately NO // status/completion marker: the job's presence-and-running state IS the in-progress signal. "trap 'self_remove' EXIT", // Pre-restart exec-ability gate: missing target → roll back and exit WITHOUT restarting. 'if [ ! -x "$EXEC_TARGET" ]; then', ` mv ${mvFlag} "$PREVIOUS" "$RUNTIME"`, " mark rolled-back", " self_remove", " exit 1", "fi", "old_pid=$(unit_pid)", '[ -n "$old_pid" ] || old_pid=0', "restart_unit", "ready=0", "new_pid=0", "attempt=0", // #1509 r8 (Finding 3): bound the verify loop by WALL CLOCK, not just attempt count. Even if // every curl takes its full --max-time, the loop exits at the deadline (~60s), so the guard's // total runtime is bounded by construction and a slow probe can never keep it running/blocking. "loop_start=$(now_s)", `loop_deadline=$((loop_start + ${GUARD_VERIFY_DEADLINE_SECONDS}))`, 'while [ "$attempt" -lt 30 ]; do', " attempt=$((attempt + 1))", ' if [ "$loop_start" != "0" ]; then now=$(now_s); [ "$now" != "0" ] && [ "$now" -ge "$loop_deadline" ] && break; fi', " new_pid=$(unit_pid)", ' [ -n "$new_pid" ] || new_pid=0', ` if [ "$new_pid" != "0" ] && [ "$new_pid" != "$old_pid" ] && curl -fsS --connect-timeout ${GUARD_CURL_CONNECT_SECONDS} --max-time ${GUARD_CURL_MAX_SECONDS} "$HEALTH_URL" | grep -Eq "$HEALTH_PATTERN"; then`, " ready=$((ready + 1))", " else", " ready=0", " fi", ' [ "$ready" -ge 2 ] && break', " sleep 2", "done", 'if [ "$ready" -ge 2 ] && [ "$new_pid" != "0" ] && [ "$new_pid" != "$old_pid" ]; then', ' rm -rf "$PREVIOUS" "$LEGACY"', " mark ok", " self_remove", " exit 0", "fi", // Restart never took or the process was not replaced: restore the previous runtime, // restart onto it, record the rollback, and fail loud. `mv ${mvFlag} "$PREVIOUS" "$RUNTIME"`, "restart_unit", "mark rolled-back", "self_remove", "exit 1", ].join("\n"); } // ───────────────────────────────────────────────────────────────────────────── // #1509 r5 — reaper: job model (§6.1), marker (§6.2), decision (§6.4, §7). // ───────────────────────────────────────────────────────────────────────────── /** The generation-scheme a guard label belongs to (#1509 r5 §6.1). */ export type GuardScheme = "r5" | "r4" | "legacy"; /** * One self-upgrade restart-guard launchd job from a SINGLE `launchctl list` snapshot (#1509 r5 §6.1), * parsed into its durable identity. PID is captured ONLY as a boolean `running` (positive PID column) * from the SAME line — never as an identity/ordering key (the r1–r3 mistake). Identity comes from the * label's genid (r5) / epoch (r4); completion comes from the per-generation marker. `base` is the * `${unit}.upgrade-guard` prefix used for own-vs-foreign scoping. */ export interface UpgradeGuardJob { /** The exact launchd label. */ label: string; /** The `${unit}.upgrade-guard` prefix (own-vs-foreign scoping key). */ base: string; /** Which generation scheme the label matches. */ scheme: GuardScheme; /** PID column is a positive integer (not "-") in the same snapshot line. */ running: boolean; /** * The PID from that same snapshot line, ONLY when running (#1509 r10 Finding 2). NEVER an * identity/ordering key (the r1–r3 mistake stands) — used solely to measure the process's * elapsed wall-clock runtime for the hung-guard bound, and every action taken on the job still * targets its exact LABEL via the manager (never a `kill `), so PID reuse can at worst * misread a duration, never touch a foreign process. */ pid?: number; /** r5 only: `s${seq}.${uuidhex}` — matches the label suffix AND the marker content token. */ genid?: string; /** r5 only: the monotonic sequence. */ seq?: number; /** r5 only: the 32-hex uuid. */ uuidhex?: string; /** r4 only: the `.g` generation digits (string) — matches the shared marker content token. */ epoch?: string; } // Anchored per-scheme label classifiers (#1509 r5 §6.1). Mutually exclusive by construction. const GUARD_R5_LABEL_RE = /^(.*\.upgrade-guard)\.s(\d+)\.([0-9a-f]{32})$/; const GUARD_R4_LABEL_RE = /^(.*\.upgrade-guard)\.g(\d+)$/; const GUARD_LEGACY_LABEL_RE = /^(.*\.upgrade-guard)$/; /** * Parse `launchctl list` output (tab-separated `PID\tStatus\tLabel` per line) into a * `{label, base, scheme, running, …}` for EVERY self-upgrade guard job. Raw/unscoped: it returns * foreign guards too (another orchestrator's, an unrelated app's) — scoping to the own-unit * base is the caller's job (see reapStaleUpgradeGuards). */ export function parseUpgradeGuardJobs(launchctlListOutput: string): UpgradeGuardJob[] { const jobs: UpgradeGuardJob[] = []; for (const line of launchctlListOutput.split("\n")) { const cols = line.split("\t"); if (cols.length < 3) continue; const label = cols[2]; if (!label) continue; const pidRaw = (cols[0] ?? "").trim(); const running = /^\d+$/.test(pidRaw) && Number(pidRaw) > 0; const pid = running ? Number(pidRaw) : undefined; let m = GUARD_R5_LABEL_RE.exec(label); if (m) { jobs.push({ label, base: m[1]!, scheme: "r5", running, pid, seq: Number(m[2]), uuidhex: m[3]!, genid: `s${m[2]}.${m[3]}` }); continue; } m = GUARD_R4_LABEL_RE.exec(label); if (m) { jobs.push({ label, base: m[1]!, scheme: "r4", running, pid, epoch: m[2]! }); continue; } m = GUARD_LEGACY_LABEL_RE.exec(label); if (m) { jobs.push({ label, base: m[1]!, scheme: "legacy", running, pid }); continue; } } return jobs; } /** * #1509 r12 (Finding 3) — "provably absent from the launchd listing" requires a WELL-FORMED listing, * not merely exit 0. r11 fed exit-0 stdout straight into the row extractor, so garbled/truncated/ * foreign-shaped output (a partial write, an error banner, binary junk) parsed to ZERO rows and read * as proof that no guard exists — in the dispatch precondition, in its confirm-absent recheck, and in * the post-dispatch fence probe — admitting a swap (or a fence-lowering rollback) beside a * live-but-invisible guard. A TRUSTED listing requires EVERY non-empty line to be either the * `PID\tStatus\tLabel` header or a `