import type { OrchestratorConfig } from "../config"; import type { ManagedAgentReport } from "../relay"; import { sanitizeFsName } from "agent-relay-sdk/fs-name"; import { shellEscape } from "agent-relay-sdk/shell-utils"; import { tmuxHasSession } from "agent-relay-sdk/tmux-utils"; import { cleanupSessionRecord } from "./supervisor"; import { currentSessionPid, findSessionRecord, isOwnerConfirmedDead, isSessionRecordAlive, loadState, loadStateOutcome, readRunnerInfo, saveState, sessionRecordLiveness, sessionRecordLivenessDetailed, sessionReportFields, sessionSupervisor, type SessionRecordLiveness, type StateLoadOutcome } from "./runtime"; import { orchestratorStateOwnerId } from "./constants"; import { reconstructOwnedLiveSessions } from "../wedged-session-reaper"; import type { SessionInfo, SessionRecord } from "./types"; // Map a recovered/adopted session record to the relay report shape. `pid` is passed // explicitly because a systemd-supervised record's current pid can differ from the // persisted one (currentSessionPid), while an adopted record simply reuses its own. function managedReportFromRecord(record: SessionRecord, pid: number): ManagedAgentReport { return { agentId: record.agentId, provider: record.provider as "claude" | "codex", workspaceMode: record.workspaceMode, workspace: record.workspace ?? (record.workspaceMode ? { mode: "shared", requestedMode: record.workspaceMode } : undefined), ...sessionReportFields(record), cwd: record.cwd, label: record.label, approvalMode: record.approvalMode || "guarded", policyName: record.policyName, spawnRequestId: record.spawnRequestId, automationRunId: record.automationRunId, pid, startedAt: record.startedAt, }; } export function listSessions(prefix: string): SessionInfo[] { return loadState() .filter((r) => r.name.startsWith(`${prefix}-`)) .map((r) => { const supervisor = sessionSupervisor(r); return { name: r.name, sessionName: r.name, pid: currentSessionPid(r), alive: isSessionRecordAlive(r), supervisor: supervisor.type, ...(supervisor.type === "systemd" && supervisor.unit ? { systemdUnit: supervisor.unit } : {}), terminalSession: r.name, terminalAvailable: tmuxHasSession(r.name, readRunnerInfo(r)?.tmuxSocket), logFile: r.logFile, }; }); } export function isSessionAlive(name: string): boolean { const record = loadState().find((r) => r.name === name); return record ? isSessionRecordAlive(record) : false; } export function managedSessionLiveness(name: string): "alive" | "dead" | "unknown" { const record = loadState().find((r) => r.name === name); return record ? sessionRecordLiveness(record) : "dead"; } // Detailed variant that also returns the systemd diagnostics read (when applicable) so // a caller reporting a dead session's exit status can reuse this same read instead of // querying `systemctl show` again later, once the unit may already be gone (#1317). export function managedSessionLivenessDetailed(name: string): SessionRecordLiveness { const record = loadState().find((r) => r.name === name); if (!record) return { liveness: "dead" }; return sessionRecordLivenessDetailed(record); } export function refreshManagedAgentReport(agent: ManagedAgentReport): ManagedAgentReport { // #1746 fwd — refresh a SPECIFIC known agent by its UNIQUE keys only. Since #1746 fwd made // selectSessionRecord "try every key" (a MISS falls through to the next, for the kill-switch), // passing policyName here would let a full-key-set miss (this agent's own session gone from state) // fall through to the policyName branch and refresh THIS agent's report with a live SIBLING policy // session's fields. The unique keys fully identify the agent; a genuine miss must return the agent // unchanged (as before), never a sibling. const record = findSessionRecord({ tmuxSession: agent.sessionName ?? agent.tmuxSession, agentId: agent.agentId, spawnRequestId: agent.spawnRequestId, }); if (!record) return agent; return { ...agent, workspaceMode: record.workspaceMode, workspace: record.workspace ?? agent.workspace ?? (record.workspaceMode ? { mode: "shared", requestedMode: record.workspaceMode } : undefined), pid: currentSessionPid(record), ...sessionReportFields(record), }; } export async function recoverExistingSessions( config: OrchestratorConfig, // #1514 (r8) — injectable seams so the persist decision below is testable without the // real global state file / tmux. Production callers pass nothing. deps: { loadOutcome?: () => StateLoadOutcome; save?: (records: SessionRecord[]) => void; ownerId?: string; reconstruct?: (input: { tmuxPrefix: string; ownerId: string }) => SessionRecord[]; } = {}, ): Promise { // #1514 — read the RAW outcome. If the state file is missing/corrupt/invalid we MUST // NOT fall through to the saveState() below: that would persist a valid empty `[]`, // erasing the loss signal the wedged-session reaper relies on to fail safe and // leaving the next spawn appending to an empty set (delayed mass-reap). Instead we // reconstruct the tracked set from live tmux reality (r3) — see the branch below. const outcome = (deps.loadOutcome ?? loadStateOutcome)(); if (outcome.status !== "loaded") { // #1514 (r3) — the state file is missing/corrupt/structurally-invalid. We must NOT // persist an empty `[]` blindly (that erases the loss signal AND leaves the next spawn // appending to an empty set, delayed-mass-reaping healthy pre-restart sessions). // Instead RECONSTRUCT the tracked set from live tmux reality: adopt the sessions // this orchestrator owns whose owning runner is still alive, so "managed" reflects // reality BEFORE any reaping and before the next spawn writes a partial record. // Sessions with a dead/absent runner are genuine orphans — deliberately left // unadopted so the reaper still reaps them. const ownerId = deps.ownerId ?? orchestratorStateOwnerId(); if (!ownerId) { // Without an owner id reconstruction can prove ownership of nothing — it never // OBSERVED reality, so persisting its empty result would erase the loss signal // on pure ignorance. Keep the state missing/corrupt (the reaper keeps skipping). console.error(`[orchestrator] Managed-session state ${outcome.status}; no owner id to reconstruct with — skipping recovery persist (fail-safe, #1514)`); return []; } let adopted: SessionRecord[]; try { adopted = (deps.reconstruct ?? reconstructOwnedLiveSessions)({ tmuxPrefix: config.tmuxPrefix, ownerId }); } catch (error) { // Reconstruction FAILED to observe reality. This covers BOTH a thrown tmux/fs error // mid-enumeration AND (#1514 r9, finding 2) a per-socket tmux failure that the // enumeration used to swallow silently: enumerateLiveManagedSessions now reports // completeness, and reconstructOwnedLiveSessions THROWS on any incomplete observation, // so an empty/partial result can never masquerade as completed reality here. This is // the one case where "nothing adopted" is not a conclusion — persisting would clobber // the loss signal (and any surviving records) with an unverified snapshot. Fail safe: // leave state unpersisted (the reaper keeps skipping) and let a later recovery/spawn // re-establish it. console.error(`[orchestrator] Managed-session state ${outcome.status}; reconstruction failed (${error}) — skipping recovery persist (fail-safe, #1514)`); return []; } // #1514 (r8, finding 2) — persist the reconstructed set EVEN WHEN EMPTY. Reconstruction // genuinely ran to completion over observed tmux reality, so an empty result is a real // conclusion: there are no adoptable owned live sessions (none exist, or every one's // owner is positively dead — a genuine orphan deliberately left for the reaper). The r7 // code refused to persist here, which left the state file missing/corrupt forever when // the ONLY survivors were dead-owner orphans: every subsequent reap cycle (and the // tombstone sweep) hit skippedNoState and the orphans leaked indefinitely. Persisting // the honest (possibly empty) snapshot restores the reaper's ability to make progress; // healthy-but-unadopted sessions remain protected by the positive-orphan gate (guard 5) // and the identity-atomic kill (guard 6). (deps.save ?? saveState)(adopted); console.error( adopted.length === 0 ? `[orchestrator] Managed-session state ${outcome.status}; reconstruction concluded with no adoptable owned live sessions — persisted empty state so the reaper can progress (#1514 r8)` : `[orchestrator] Managed-session state ${outcome.status}; reconstructed ${adopted.length} owned live session(s) from tmux reality (#1514)`, ); return adopted.map((record) => managedReportFromRecord(record, record.pid)); } const records = outcome.records.filter((r) => r.name.startsWith(`${config.tmuxPrefix}-`)); const managed: ManagedAgentReport[] = []; const alive: SessionRecord[] = []; for (const record of records) { // #1514 (r6) — cleanup (which writes the identity-bound tombstone + deletes runner-info) is // authorized ONLY by the SINGLE fail-closed gate, the same one the reaper consults. Only // positive, identity-bound proof of death removes a session; EPERM / ambiguous / systemd- // unavailable / missing runner-info ⇒ NOT confirmed dead ⇒ preserve it (tracked, runner-info // intact) — never the two-state "dead" that used to false-cleanup a live EPERM session (the // r5 hole). A genuinely-dead runner still has runner-info naming its dead pid at this point, // so it IS confirmed dead here and cleaned, writing the tombstone that drives the real reap. if (isOwnerConfirmedDead({ name: record.name })) { console.error(`[orchestrator] Stale session: ${record.name} (owning runner confirmed dead) — removing`); cleanupSessionRecord(record); continue; } if (sessionRecordLiveness(record) === "unknown") { console.error(`[orchestrator] Session liveness unknown: ${record.name} (pid ${record.pid}) — preserving`); } const pid = currentSessionPid(record); const updatedRecord = { ...record, pid }; alive.push(updatedRecord); managed.push(managedReportFromRecord(record, pid)); console.error(`[orchestrator] Recovered existing session: ${record.name} (pid ${record.pid})`); } // Merge rather than overwrite: only replace the records this recovery actually // inspected, so a session added concurrently (or owned by another prefix) is // not erased by writing back a pre-filtered snapshot. const processedNames = new Set(records.map((r) => r.name)); const untouched = loadState().filter((r) => !processedNames.has(r.name)); saveState([...untouched, ...alive]); return managed; } export function managedAgentId(config: OrchestratorConfig, provider: string, label: string): string { const cleanHost = sanitizeFsName(config.hostname, { replacement: "-", lowercase: true }); const cleanLabel = sanitizeFsName(label, { replacement: "-", lowercase: true }); return `${cleanHost}-${provider}-${cleanLabel}-${crypto.randomUUID().slice(0, 8)}`; } // Shared shell-quoting; re-exported so `./spawn` consumers + tests resolve it. export { shellEscape };