import { errMessage, isPathWithinBase, isRecord, normalizeAgentLifecycle, normalizeWorkspaceMode, Semaphore } from "agent-relay-sdk"; import { renderTmuxSessionTarget } from "agent-relay-sdk/tmux-utils"; import { fireAndForget } from "./async-guard"; import { getAllManifests, getManifest } from "agent-relay-providers"; import { SCHEDULER_COMMAND_MAX_CONCURRENT, SCHEDULER_COMMAND_MAX_OUTPUT_BYTES, type OrchestratorConfig } from "./config"; import { checkHostHeadroom, isHostAdmissionDisabled, queuedSpawnFromCommand, resolveAdmissionMaxWaitMs, resolveAdmissionPollMs, resolveMinFreeMemMb, waitForHostHeadroom, type HostHeadroomResult } from "./host-admission"; import type { ManagedAgentReport, RelayClient, RelayCommand } from "./relay"; import type { QueuedSpawn } from "agent-relay-sdk"; import { handleSelfUpgrade } from "./self-upgrade"; import { readLocalProviderConfigs } from "./provider-config-migration"; import { findSessionRecord, isSessionAlive, readRunnerInfo, spawnAgent, stopSession, type SpawnOptions } from "./spawn"; import { cleanupWorkspace, discardRecoveryBranch, handleReviewRefCommand, idleRefreshWorktree, mergeWorkspace, mergeWorkspacePlainGit, pruneWorktrees, reconcileWorkspace, refreshWorkspaceDeps, runClaimGatedPublishReview, workspacesRoot } from "./workspace-probe"; import { withMergePhaseTimeout } from "./workspace-probe/merge-timeouts"; import { withRepoLock } from "./repo-lock"; import { armWorkspacePrAutoMerge, mergeWorkspacePr, refreshWorkspacePrBranch } from "./workspace-pr"; import type { WorkspaceMergeResult } from "agent-relay-sdk"; import { execProcess } from "./process"; import { mergeCommandStatus, prepareWorkspaceMergeResultForReport } from "./command-result-report"; // #1020 (B2) — command types that mutate a physical repo's git refs / worktrees. These take the // per-repoRoot lock (see repo-lock.ts) so they can't interleave with the background spawn's // acquisition + `git worktree add` on the same checkout. Read-only or worktree-filesystem-only // commands (e.g. workspace.deps-refresh) are omitted so a slow dep install can't stall spawns. const REPO_MUTATING_COMMANDS = new Set([ "workspace.cleanup", "workspace.reconcile", "workspace.merge", "workspace.pr-merge", "workspace.pr-refresh", "workspace.pr-arm-auto-merge", "workspace.recovery-branch-discard", "workspace.idle-refresh", "workspace.prune", ]); interface ControlHandler { handleCommand(command: RelayCommand): Promise; getManagedAgents(): ManagedAgentReport[]; setManagedAgents(agents: ManagedAgentReport[]): void; // #930/#880 — spawn commands dispatch in the background (claimed synchronously, // then run past registration under the concurrency cap). These expose that // in-flight background work so callers/tests can observe and settle it. spawnsInFlight(): number; settleSpawns(): Promise; } const DEFAULT_MAX_CONCURRENT_SPAWNS = 2; const TERMINAL_COMMAND_STATUSES = new Set(["succeeded", "failed", "timed_out", "rejected", "canceled"]); // #930/#880 — per-host spawn concurrency cap. Bursts of spawns racing worktree // materialization + port churn during the registration window are what surface // as `EADDRINUSE`/"operation was aborted" on the loser (#880). Bound how many // spawns materialize at once. Parsed once at handler construction; invalid or // out-of-range values fall back to the safe default rather than throwing. export function resolveMaxConcurrentSpawns(env: NodeJS.ProcessEnv = process.env): number { const raw = env.AGENT_RELAY_MAX_CONCURRENT_SPAWNS; if (raw === undefined || raw.trim() === "") return DEFAULT_MAX_CONCURRENT_SPAWNS; const parsed = Number(raw); if (!Number.isInteger(parsed) || parsed < 1) return DEFAULT_MAX_CONCURRENT_SPAWNS; return parsed; } // #930/#880 — how long a spawn slot is held while we wait for the child to // register (bind its local control/ws server — the contended resource). This is // a fail-safe upper bound only: a child that never registers must not wedge a // slot forever, so we release after this deadline even if it hasn't appeared. // The happy path releases as soon as the child's runner-info file materializes, // well under this ceiling. const DEFAULT_SPAWN_REGISTRATION_TIMEOUT_MS = 30_000; const SPAWN_REGISTRATION_POLL_MS = 100; // #1201 F3 — post-registration liveness settle. waitForManagedRegistration returns on the FIRST // `registeredAt` write; a successor that crashes microseconds later would, for a NON-managed agent // (no policy → no backoff respawn), be lost the instant we stop the predecessor. Before tearing // down the predecessor we wait this beat and confirm the successor is STILL alive. Managed agents // have backoff respawn as a net; non-managed must never be strandable. const DEFAULT_SPAWN_LIVENESS_SETTLE_MS = 300; export function resolveSpawnRegistrationTimeoutMs(env: NodeJS.ProcessEnv = process.env): number { const raw = env.AGENT_RELAY_SPAWN_REGISTRATION_TIMEOUT_MS; if (raw === undefined || raw.trim() === "") return DEFAULT_SPAWN_REGISTRATION_TIMEOUT_MS; const parsed = Number(raw); if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_SPAWN_REGISTRATION_TIMEOUT_MS; return parsed; } export function resolveMaxConcurrentSchedulerCommands(env: NodeJS.ProcessEnv = process.env): number { const raw = env.AGENT_RELAY_SCHEDULER_COMMAND_MAX_CONCURRENT; if (raw === undefined || raw.trim() === "") return SCHEDULER_COMMAND_MAX_CONCURRENT; const parsed = Number(raw); if (!Number.isInteger(parsed) || parsed < 1) return SCHEDULER_COMMAND_MAX_CONCURRENT; return parsed; } export function resolveSchedulerCommandMaxOutputBytes(env: NodeJS.ProcessEnv = process.env): number { const raw = env.AGENT_RELAY_SCHEDULER_COMMAND_MAX_OUTPUT_BYTES; if (raw === undefined || raw.trim() === "") return SCHEDULER_COMMAND_MAX_OUTPUT_BYTES; const parsed = Number(raw); if (!Number.isSafeInteger(parsed) || parsed < 4096) return SCHEDULER_COMMAND_MAX_OUTPUT_BYTES; return parsed; } /** Awaits a spawn slot until the child registers, or resolves false at the fail-safe deadline. */ export type WaitForRegistration = (agent: ManagedAgentReport, timeoutMs: number) => Promise; // Registration is explicit in the runner-info file. `controlUrl` appears before the // Relay bus register completes, so using file existence alone creates a false-success // window where a restart can kill the predecessor before the successor owns its Relay // identity. `registeredAt` is written only after registerWithinDeadline resolves. async function waitForManagedRegistration(agent: ManagedAgentReport, timeoutMs: number): Promise { const session = agent.sessionName ?? agent.tmuxSession; if (!session) return false; const deadline = Date.now() + timeoutMs; for (;;) { const record = findSessionRecord({ tmuxSession: session, agentId: agent.agentId }); if (record && readRunnerInfo(record)?.registeredAt) return true; if (Date.now() >= deadline) return false; await Bun.sleep(SPAWN_REGISTRATION_POLL_MS); } } interface ControlHandlerDeps { // Injectable spawn dispatch — defaults to the real spawnAgent. Only tests // override it, to gate dispatch and observe the concurrency bound directly. spawnAgent?: typeof spawnAgent; // Override the cap directly, bypassing the env parse (tests). maxConcurrentSpawns?: number; // Injectable registration wait — defaults to polling the child's runner info // file. Tests override it to gate the registration window explicitly. waitForRegistration?: WaitForRegistration; // Override the fail-safe registration timeout, bypassing the env parse (tests). registrationTimeoutMs?: number; // Injectable stop dispatch — production uses tmux/system process teardown. stopSession?: typeof stopSession; maxConcurrentSchedulerCommands?: number; schedulerCommandMaxOutputBytes?: number; // #1201 F3 — injectable post-registration liveness settle. Defaults to sleeping settleMs then // checking the successor session is still alive. Tests override it to simulate a register-then-crash. settleRegistration?: (agent: ManagedAgentReport, settleMs: number) => Promise; registrationSettleMs?: number; // #1201 F2 — injectable safe-point re-check. Defaults to relay.recheckSelfRestartSafePoint. recheckSafePoint?: (agentId: string) => Promise<{ ok: boolean; reason?: string }>; // #893 — injectable host headroom check. Defaults to a real os.freemem()/loadavg() read. // Tests override it to drive the admission gate deterministically. checkHostHeadroom?: () => HostHeadroomResult; hostAdmissionDisabled?: boolean; admissionMaxWaitMs?: number; admissionPollMs?: number; } export function createControlHandler( config: OrchestratorConfig, relay: RelayClient, deps: ControlHandlerDeps = {}, ): ControlHandler { let managedAgents: ManagedAgentReport[] = []; const dispatchSpawn = deps.spawnAgent ?? spawnAgent; const dispatchStop = deps.stopSession ?? stopSession; // In-memory counter is correct here: a slot only needs to live within this // orchestrator process, and a crash resets in-flight spawns anyway (#881 Q1). const spawnSlots = new Semaphore(deps.maxConcurrentSpawns ?? resolveMaxConcurrentSpawns()); const waitForRegistration = deps.waitForRegistration ?? waitForManagedRegistration; const registrationTimeoutMs = deps.registrationTimeoutMs ?? resolveSpawnRegistrationTimeoutMs(); const registrationSettleMs = deps.registrationSettleMs ?? DEFAULT_SPAWN_LIVENESS_SETTLE_MS; const settleRegistration = deps.settleRegistration ?? (async (agent, settleMs) => { if (settleMs > 0) await Bun.sleep(settleMs); const session = agent.sessionName ?? agent.tmuxSession; return session ? isSessionAlive(session) : false; }); const recheckSafePoint = deps.recheckSafePoint ?? ((agentId: string) => relay.recheckSelfRestartSafePoint(agentId)); // #893 — per-host memory/load admission gate. Runs once per spawn, right before launch, so a // spawn that would tip the host into swap-thrash queues (bounded poll/retry) instead of firing. const headroomCheck = deps.checkHostHeadroom ?? checkHostHeadroom; const admissionDisabled = deps.hostAdmissionDisabled ?? isHostAdmissionDisabled(); const admissionMaxWaitMs = deps.admissionMaxWaitMs ?? resolveAdmissionMaxWaitMs(); const admissionPollMs = deps.admissionPollMs ?? resolveAdmissionPollMs(); const schedulerCommandSlots = new Semaphore(deps.maxConcurrentSchedulerCommands ?? resolveMaxConcurrentSchedulerCommands()); const schedulerCommandMaxOutputBytes = deps.schedulerCommandMaxOutputBytes ?? resolveSchedulerCommandMaxOutputBytes(); // Backgrounded spawn tasks: each is claimed synchronously (status → accepted) // then runs to completion off the poll tick, holding a slot across the child's // registration window. Tracked so tests/shutdown can observe and settle them. const backgroundSpawns = new Set>(); const backgroundCommands = new Set>(); // #1513 — spawns currently held back by the host headroom admission gate, keyed by command id. // Published on the roster PATCH so the dashboard shows a "queued: waiting for host memory" row and // the relay notifies the spawner. Entries are added on the first queued poll and removed once the // spawn is admitted or times out (see the onQueued/onSettled wiring in runSpawnToRegistration). const queuedSpawns = new Map(); const minFreeMemMb = resolveMinFreeMemMb(); function publishQueuedSpawns(): void { // Best-effort — a failed roster write is retried by the reconnect resync (which carries the held // snapshot). Never let a roster-push rejection escape into the backgrounded spawn task. void Promise.resolve(relay.updateManagedAgents(managedAgents, [], [...queuedSpawns.values()])).catch(() => undefined); } async function spawnManagedAgent(opts: SpawnOptions, action = "Spawned"): Promise { const agent = await dispatchSpawn(opts, config); managedAgents.push(agent); console.error(`[orchestrator] ${action} ${opts.provider} agent: ${agent.tmuxSession}`); return agent; } async function spawnAndWaitForRegistration(opts: SpawnOptions, action = "Spawned", requireRegistration = false): Promise { const agent = await spawnManagedAgent(opts, action); const registered = await waitForRegistration(agent, registrationTimeoutMs); if (!registered) { if (requireRegistration) { const candidateSession = agent.sessionName ?? agent.tmuxSession; if (candidateSession) { await dispatchStop(candidateSession, config, "successor failed to register", false, 5_000).catch(() => undefined); managedAgents = managedAgents.filter((item) => (item.sessionName ?? item.tmuxSession) !== candidateSession); } throw new Error(`successor ${candidateSession ?? agent.agentId} did not register within ${registrationTimeoutMs}ms`); } console.error(`[orchestrator] spawn ${agent.tmuxSession} did not register within ${registrationTimeoutMs}ms; releasing slot (fail-safe)`); } return agent; } // #1201 F2/F3 — reconcile an aborted self-restart teardown by removing the SUCCESSOR (keeping // the predecessor). Best-effort stop; drop it from the managed roster so we don't report a // generation we just killed. async function tearDownSuccessor(successor: ManagedAgentReport, reason: string): Promise { const successorSession = successor.sessionName ?? successor.tmuxSession; if (!successorSession) return; await dispatchStop(successorSession, config, reason, false, 5_000).catch(() => undefined); managedAgents = managedAgents.filter((item) => (item.sessionName ?? item.tmuxSession) !== successorSession); } async function markSpawnCommandFailed(command: RelayCommand, error: unknown): Promise { console.error(`[orchestrator] spawn dispatch failed: ${errMessage(error)}`); // #930 — the `failed` status write can itself reject during a relay blip. Wrap it so // a failed status write can't re-throw out of the backgrounded task (belt-and-suspenders // with the `.catch` in dispatchSpawnCommand). The command may strand non-terminal until // the TTL sweep / watchdog reconciles it, but the task settles cleanly regardless. try { await relay.updateCommand(command.id, "failed", undefined, errMessage(error)); } catch (statusError) { console.error(`[orchestrator] failed to mark spawn command failed: ${errMessage(statusError)}`); } } // #930/#880 — dispatch a spawn command concurrently, up to the per-host cap. // // The command is CLAIMED (status → accepted) synchronously here, BEFORE we // return, so the next pollCommands() tick can never re-return this still-pending // row and double-dispatch it. The slot itself is taken in the BACKGROUND via // acquire() (strict FIFO — no starvation), so the poll tick is never blocked // waiting for a free slot, preserving the #864 single-flight of the tick. // // The background task holds its slot across the child's registration window // (#880's contended ws-bind resource) and frees it in `finally` on EVERY path: // registration success, spawn failure, or the fail-safe registration timeout. async function dispatchSpawnCommand(command: RelayCommand): Promise { await relay.updateCommand(command.id, "accepted"); // #930 — guard the backgrounded task against unhandled rejection. runSpawnToRegistration // settles its own errors, but a relay blip in the ≤30s in-flight window can reject the // status write in its catch block too; without a terminal `.catch` here that escapes as // an unhandledRejection (log spam, or a crash if the runtime exits on it). Track the // GUARDED promise so settleSpawns() awaits a promise that never rejects. const task = runSpawnToRegistration(command).catch((error) => { console.error(`[orchestrator] backgrounded spawn task error (swallowed): ${errMessage(error)}`); }); backgroundSpawns.add(task); void fireAndForget("Background spawn bookkeeping", task.finally(() => backgroundSpawns.delete(task))); return true; } async function runSpawnToRegistration(command: RelayCommand): Promise { // Blocks FIFO until a slot frees — this is what bounds concurrent spawns to N. await spawnSlots.acquire(); let launched = false; try { const current = await relay.getCommand(command.id); if (!current || TERMINAL_COMMAND_STATUSES.has(current.status)) { console.error(`[orchestrator] skipping spawn command ${command.id}: ${current ? `status is ${current.status}` : "command not found"}`); return; } await relay.updateCommand(command.id, "running"); try { // #893 — host memory/load admission gate; see host-admission.ts. // #1513 — surface the queued state: publish a "queued for host memory" roster row (and let the // relay notify the spawner) while held back, updating the reason/free-memory as headroom moves, // and clear it once the spawn is admitted or the wait ceiling is exceeded. const queuedAt = Date.now(); await waitForHostHeadroom({ disabled: admissionDisabled, maxWaitMs: admissionMaxWaitMs, pollMs: admissionPollMs, check: headroomCheck, log: (message) => console.error(`[orchestrator] ${message}`), onQueued: (result) => { queuedSpawns.set(command.id, queuedSpawnFromCommand(command.params, result, queuedAt, minFreeMemMb, admissionMaxWaitMs)); publishQueuedSpawns(); }, onSettled: () => { if (queuedSpawns.delete(command.id)) publishQueuedSpawns(); }, }); await spawnAndWaitForRegistration(spawnOptionsFromControl(command.params, config)); launched = true; } catch (error) { await markSpawnCommandFailed(command, error); return; } // #956 — from this point the child process launched. A relay blip while // recording success is a settle failure, not a spawn failure; do not flip // the command to failed and notify the parent while the child may register. await relay.updateCommand(command.id, "succeeded", { managedAgents }); await relay.updateManagedAgents(managedAgents); } catch (error) { if (launched) { console.error(`[orchestrator] spawn command settle failed after launch: ${errMessage(error)}`); } else { await markSpawnCommandFailed(command, error); } } finally { spawnSlots.release(); } } async function settleSpawns(): Promise { while (backgroundSpawns.size > 0) { await Promise.all([...backgroundSpawns]); } } async function handleShutdown(ctrl: Record, restart = false): Promise> { const current = managedAgentShutdownTarget(managedAgents, ctrl); // #1746 RC-1(c) — when the in-memory managedAgents report has no match (an orchestrator that // restarted and lost its report cache, or a report/record divergence), fall back to the durable // on-disk session state before giving up. findSessionRecord (post-fix) tries every key, so a live // session the report missed can still be resolved and force-killed rather than silently skipped. const fallbackRecord = current ? undefined : findSessionRecord({ agentId: typeof ctrl.agentId === "string" && ctrl.agentId ? ctrl.agentId : undefined, spawnRequestId: typeof ctrl.spawnRequestId === "string" && ctrl.spawnRequestId ? ctrl.spawnRequestId : undefined, policyName: typeof ctrl.policyName === "string" && ctrl.policyName ? ctrl.policyName : undefined, tmuxSession: typeof ctrl.sessionName === "string" && ctrl.sessionName ? ctrl.sessionName : typeof ctrl.tmuxSession === "string" && ctrl.tmuxSession ? ctrl.tmuxSession : undefined, }); const session = current?.sessionName ?? current?.tmuxSession ?? fallbackRecord?.name; const restartSpawn = isRecord(ctrl.restartSpawn) ? ctrl.restartSpawn : undefined; if (!session) { let restarted: ManagedAgentReport | undefined; if (restart && restartSpawn) { await spawnSlots.acquire(); try { restarted = await spawnAndWaitForRegistration(spawnOptionsFromRestartSource(restartSpawn, config), "Restarted", true); } finally { spawnSlots.release(); } } // #1746 RC-1(c) — the target resolved to NOTHING and (for a restart) no successor was spawned: // dispatchStop never ran, so we CANNOT claim the process was stopped. Signal `targetNotFound` so // the caller marks the command FAILED (a red terminal outcome the dashboard/watchdog can act on) // instead of the old inert `succeeded`, which deregistered live agents and wrote a false // `agent.exited`. A restart that DID respawn a successor is a legitimate success. return { stopped: false, wasRunning: false, restart, restarted: Boolean(restarted), ...(restarted ? { agent: restarted } : { targetNotFound: true }), policyName: ctrl.policyName, spawnRequestId: ctrl.spawnRequestId, }; } // A managed restart carries a fresh spawnRequestId in restartSpawn — keep it. // Falling back to the live agent's params would reuse the stale id and break // relay correlation, so drop it and let spawnAgent assign a new identity. const restartSource = (restartSpawn ?? (current ? { ...current, spawnRequestId: undefined } : undefined)) as Record | undefined; let restarted: ManagedAgentReport | undefined; if (restart && restartSpawn && restartSource) { await spawnSlots.acquire(); try { // Transactional swap: the successor must complete Relay registration before the // predecessor session is touched. A spawn/registration failure throws with the old // process still alive and the durable continuation artifact still fetchable. restarted = await spawnAndWaitForRegistration(spawnOptionsFromRestartSource(restartSource, config), "Restarted", true); } finally { spawnSlots.release(); } // #1201 F3 — liveness settle: confirm the successor is STILL alive a beat after its first // `registeredAt` write, BEFORE the predecessor teardown becomes irreversible. A successor that // crashes microseconds after registering would otherwise strand a non-managed agent (no // policy → no backoff respawn). On failure: tear down the (dead) successor, keep predecessor. if (!(await settleRegistration(restarted, registrationSettleMs))) { await tearDownSuccessor(restarted, "successor died during post-registration liveness settle"); throw new Error(`successor ${restarted.sessionName ?? restarted.tmuxSession} did not stay alive after registration settle; predecessor kept`); } // #1201 F2 — TOCTOU safe-point re-check for a SELF-restart, as late as possible before the // kill. register-before-kill left the predecessor running after it got {ok:true}, so it could // have created WIP / spawned a child / taken a merge lease in the window. Reconciliation on a // now-violated safe-point: tear down the SUCCESSOR and KEEP the predecessor — never two live // generations, never a lost agent. The caller's own live process simply keeps running; the // failed command records why. Fail-closed: an unavailable re-check is treated as a violation. if (isRecord(ctrl.selfRestart) && typeof ctrl.agentId === "string") { const recheck = await recheckSafePoint(ctrl.agentId); if (!recheck.ok) { await tearDownSuccessor(restarted, `self-restart safe-point re-check failed: ${recheck.reason ?? "unknown"}`); throw new Error(`self-restart aborted at teardown: ${recheck.reason ?? "safe-point re-check failed"}; predecessor kept, successor torn down`); } } } const result = await dispatchStop(session, config, typeof ctrl.reason === "string" ? ctrl.reason : restart ? "restart" : "shutdown", ctrl.graceful !== false, shutdownTimeoutMs(ctrl)); managedAgents = managedAgents.filter((agent) => (agent.sessionName ?? agent.tmuxSession) !== session); if (restart && !restartSpawn && restartSource) { await spawnSlots.acquire(); try { restarted = await spawnAndWaitForRegistration(spawnOptionsFromRestartSource(restartSource, config), "Restarted"); } finally { spawnSlots.release(); } } return { ...result, restart, restarted: Boolean(restarted), ...(restarted ? { agent: restarted } : {}), policyName: ctrl.policyName, spawnRequestId: ctrl.spawnRequestId, sessionName: session, tmuxSession: session, }; } async function handleCommand(command: RelayCommand): Promise { // #930/#880 — spawns dispatch concurrently in the background under the // per-host cap (see dispatchSpawnCommand). Every other command is handled // inline, synchronously with the poll tick, exactly as before. if (command.type === "agent.spawn") return dispatchSpawnCommand(command); if (command.type === "scheduler.command") return dispatchSchedulerCommand(command); return handleNonSpawnCommand(command); } async function dispatchSchedulerCommand(command: RelayCommand): Promise { await relay.updateCommand(command.id, "accepted"); const startedAt = Date.now(); const run = (async () => { await schedulerCommandSlots.acquire(); try { await relay.updateCommand(command.id, "running"); const commandText = typeof command.params.command === "string" ? command.params.command : ""; const cwd = typeof command.params.cwd === "string" ? command.params.cwd : config.baseDir; if (!commandText.trim()) throw new Error("scheduler.command missing command"); if (!isPathWithinBase(cwd, config.baseDir)) throw new Error("scheduler.command cwd escapes orchestrator baseDir"); const timeoutMs = typeof command.params.timeoutMs === "number" && Number.isSafeInteger(command.params.timeoutMs) && command.params.timeoutMs > 0 ? command.params.timeoutMs : undefined; const env = isRecord(command.params.env) ? Object.fromEntries(Object.entries(command.params.env).filter((entry): entry is [string, string] => typeof entry[1] === "string")) : undefined; const result = await execProcess(["bash", "-lc", commandText], { cwd, env: { ...process.env, ...(env ?? {}) }, timeoutMs, timeoutLabel: `scheduler command ${command.id}`, trimStdout: false, trimStderr: false, maxOutputBytes: schedulerCommandMaxOutputBytes, // #968 hole 2 — reap the whole tree on every exit path so a backgrounded/daemonized // child (e.g. `setsid … &`) can't survive after the command returns. reapProcessGroup: true, }); const finishedAt = Date.now(); await relay.updateCommand(command.id, result.ok ? "succeeded" : result.timedOut ? "timed_out" : "failed", { command: commandText, cwd, startedAt, finishedAt, durationMs: finishedAt - startedAt, exitCode: result.exitCode, success: result.ok, timedOut: result.timedOut === true, stdoutTail: result.stdout, stderrTail: result.stderr, outputTruncated: result.outputTruncated === true, outputLimitExceeded: result.outputLimitExceeded === true, }, result.ok ? undefined : result.stderr || `command exited ${result.exitCode ?? "without code"}`); } catch (error) { await relay.updateCommand(command.id, "failed", { startedAt, finishedAt: Date.now(), durationMs: Date.now() - startedAt, }, errMessage(error)); } finally { schedulerCommandSlots.release(); } })(); backgroundCommands.add(run); run.finally(() => backgroundCommands.delete(run)); return true; } async function handleNonSpawnCommand(command: RelayCommand): Promise { // #1020 (B2) — a repo-root-mutating command takes the physical-repo lock so it can't interleave // with the concurrently-dispatched background spawn (acquisition + `git worktree add`) on the // same checkout. Non-spawn commands are already mutually serialized by the single-flight poll // tick; this closes the one carve-out (agent.spawn). Commands without a repoRoot run unwrapped. const repoRoot = typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined; if (repoRoot && REPO_MUTATING_COMMANDS.has(command.type)) { return withRepoLock(repoRoot, () => runNonSpawnCommand(command)); } return runNonSpawnCommand(command); } async function runNonSpawnCommand(command: RelayCommand): Promise { // #1452 round-11 HIGH#1 — the review publish force-pushes to origin, so it is gated on an ATOMIC // claim (helper in workspace-probe/review-ref): push ONLY IF the claim won, so a reconciler cancel // (round closed) provably precludes a late orphaning push. The ONLY command gated on the claim — // all others keep the generic accepted→running flow, confining the CAS blast radius to publish. if (command.type === "workspace.publish-review") return runClaimGatedPublishReview(command, relay, managedAgents); // #1201 F4 — at-most-once guard for restart/shutdown, mirroring runSpawnToRegistration's // terminal-status re-fetch. A redelivered agent.restart (e.g. relay reconcile after an // orchestrator bounce) must not spawn a SECOND successor sharing the predecessor's fresh // spawnRequestId. If the command already reached a terminal state, this is a replay — skip it. if (command.type === "agent.restart" || command.type === "agent.shutdown") { const current = await relay.getCommand(command.id); if (current && TERMINAL_COMMAND_STATUSES.has(current.status)) { console.error(`[orchestrator] skipping ${command.type} ${command.id}: already ${current.status} (replay guard)`); return true; } } await relay.updateCommand(command.id, "accepted"); await relay.updateCommand(command.id, "running"); try { if (command.type === "agent.shutdown" || command.type === "agent.restart") { const result = await handleShutdown(command.params, command.type === "agent.restart"); // #1746 RC-1(c) — a shutdown is a KILL-SWITCH: never report success unless the target was // resolved AND is confirmed gone. Two outcomes fail the command with a red terminal reason so // the dashboard/watchdog can escalate (and no false `agent.exited` is written server-side): // - target-not-found: nothing matched, dispatchStop never ran (the surviving-agent bug); // - kill-failed: the session was resolved but its process survived graceful+forceful stop. // Restart keeps its existing settlement — its successor spawn + predecessor teardown are // reconciled by the restart path itself, not by this kill-switch gate. const shutdownFailure = command.type === "agent.shutdown" ? (result.targetNotFound === true ? "target-not-found: no live session, agent, spawn, or policy matched the shutdown request" : result.wasRunning === true && result.stopped !== true ? "kill-failed: target process survived graceful and forceful stop" : undefined) : undefined; if (shutdownFailure) { await relay.updateCommand(command.id, "failed", result, shutdownFailure); } else { await relay.updateCommand(command.id, "succeeded", result); } } else if (command.type === "workspace.cleanup") { const result = await cleanupWorkspace({ id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined, worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined, branch: typeof command.params.branch === "string" ? command.params.branch : undefined, baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined, baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined, deleteBranch: command.params.deleteBranch !== false, force: command.params.force === true, reason: typeof command.params.reason === "string" ? command.params.reason : undefined, workspacesRoot: workspacesRoot(config.baseDir), }); // #1452 — the review ref (if any) is dropped inside cleanupWorkspace itself, host-side. await relay.updateCommand(command.id, "succeeded", result); } else if (command.type === "workspace.reconcile") { const result = await reconcileWorkspace({ id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined, worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined, branch: typeof command.params.branch === "string" ? command.params.branch : undefined, baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined, baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined, }); await relay.updateCommand(command.id, "succeeded", result); } else if (command.type === "workspace.merge") { let result: WorkspaceMergeResult; try { const mergeInput = { id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined, worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined, branch: typeof command.params.branch === "string" ? command.params.branch : undefined, expectedHeadSha: typeof command.params.expectedHeadSha === "string" ? command.params.expectedHeadSha : undefined, baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined, baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined, strategy: command.params.strategy === "pr" || command.params.strategy === "rebase-ff" || command.params.strategy === "auto" ? command.params.strategy : undefined, landMechanism: command.params.landMechanism === "plain-git" ? "plain-git" : "managed", deleteBranch: command.params.deleteBranch !== false, push: command.params.push !== false, prTitle: typeof command.params.prTitle === "string" ? command.params.prTitle : undefined, prBody: typeof command.params.prBody === "string" ? command.params.prBody : undefined, autoMerge: command.params.autoMerge === "on-green" || command.params.autoMerge === "on-approval" || command.params.autoMerge === "manual" ? command.params.autoMerge : undefined, gateLevel: command.params.gateLevel === "subset" || command.params.gateLevel === "none" ? command.params.gateLevel : undefined, gateReason: typeof command.params.gateReason === "string" ? command.params.gateReason : undefined, gateRequestedBy: typeof command.params.gateRequestedBy === "string" ? command.params.gateRequestedBy : undefined, prLanded: isRecord(command.params.prLanded) ? { sha: typeof command.params.prLanded.sha === "string" ? command.params.prLanded.sha : undefined, subject: typeof command.params.prLanded.subject === "string" ? command.params.prLanded.subject : undefined, } : undefined, } as const; result = await withMergePhaseTimeout("total", (signal) => mergeInput.landMechanism === "plain-git" ? mergeWorkspacePlainGit({ ...mergeInput, signal }) : mergeWorkspace({ ...mergeInput, signal })); } catch (err) { const branch = typeof command.params.branch === "string" ? command.params.branch : undefined; const baseRef = typeof command.params.baseRef === "string" ? command.params.baseRef : undefined; result = { workspaceId: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, strategy: command.params.strategy === "pr" ? "pr" : "rebase-ff", merged: false, status: "review_requested", ...(branch ? { branch } : {}), ...(baseRef ? { baseRef } : {}), error: errMessage(err), }; console.error(`[orchestrator] workspace.merge failed before completion: ${result.error}`); } // #638 — settle `failed` (carrying the error) for a no-op merge instead of // `succeeded`; see mergeCommandStatus. // #924 — the merge OUTCOME is now definitive: mergeWorkspace settles its own throws // into `result` at the catch above, so `result` already reflects success or failure. // A transport error while REPORTING it (the SDK http client's 10s abort firing on a // slow settle-response — "The operation was aborted.") must NOT cascade into the outer // catch's blanket `failed` re-stamp: that stamped a genuinely-successful fast-forward // land as failed/aborted WHILE result.merged=true — the self-contradictory record #924. // The relay already committed the report in the common (slow-response) case; in the rare // lost-write case the command strands non-terminal and the relay watchdog / TTL reconcile // (recoverAgedInFlightWorkspaces, the #1025 A7 timed_out→succeeded correction) settles it. // Same belt-and-suspenders as the spawn path (#930, markSpawnCommandFailed). try { const report = prepareWorkspaceMergeResultForReport(result); await relay.updateCommand(command.id, mergeCommandStatus(report), report as unknown as Record, report.error); } catch (reportError) { console.error(`[orchestrator] workspace.merge result report failed for ${command.id} (not re-stamping as failed — relay reconcile owns recovery): ${errMessage(reportError)}`); } } else if (command.type === "workspace.pr-arm-auto-merge") { const rawPrNumber = command.params.prNumber; const result = await armWorkspacePrAutoMerge({ id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined, worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined, branch: typeof command.params.branch === "string" ? command.params.branch : undefined, prNumber: typeof rawPrNumber === "number" && Number.isSafeInteger(rawPrNumber) ? rawPrNumber : undefined, prUrl: typeof command.params.prUrl === "string" ? command.params.prUrl : undefined, }); await relay.updateCommand(command.id, result.autoMergeArmed ? "succeeded" : "failed", result as unknown as Record, result.error); } else if (command.type === "workspace.pr-merge") { const rawPrNumber = command.params.prNumber; const result = await mergeWorkspacePr({ id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined, worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined, branch: typeof command.params.branch === "string" ? command.params.branch : undefined, prNumber: typeof rawPrNumber === "number" && Number.isSafeInteger(rawPrNumber) ? rawPrNumber : undefined, prUrl: typeof command.params.prUrl === "string" ? command.params.prUrl : undefined, }); await relay.updateCommand(command.id, result.relayMerged ? "succeeded" : "failed", result as unknown as Record, result.error); } else if (command.type === "workspace.pr-refresh") { const rawPrNumber = command.params.prNumber; const result = await refreshWorkspacePrBranch({ id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined, worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined, branch: typeof command.params.branch === "string" ? command.params.branch : undefined, prNumber: typeof rawPrNumber === "number" && Number.isSafeInteger(rawPrNumber) ? rawPrNumber : undefined, prUrl: typeof command.params.prUrl === "string" ? command.params.prUrl : undefined, }); await relay.updateCommand(command.id, result.prRefreshed ? "succeeded" : "failed", result as unknown as Record, result.error); } else if (command.type === "workspace.deps-refresh") { const result = await refreshWorkspaceDeps( typeof command.params.repoRoot === "string" ? command.params.repoRoot : "", typeof command.params.worktreePath === "string" ? command.params.worktreePath : "", { checkOnly: command.params.checkOnly === true }, ); await relay.updateCommand(command.id, "succeeded", { workspaceId: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, ...result }); } else if (command.type === "workspace.recovery-branch-discard") { const result = await discardRecoveryBranch({ repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined, branch: typeof command.params.branch === "string" ? command.params.branch : undefined, baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined, baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined, force: command.params.force === true, }); await relay.updateCommand(command.id, "succeeded", result as unknown as Record); } else if (command.type === "workspace.idle-refresh") { const result = await idleRefreshWorktree({ id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined, worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined, branch: typeof command.params.branch === "string" ? command.params.branch : undefined, baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined, baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined, }); await relay.updateCommand(command.id, result.error ? "failed" : "succeeded", result as unknown as Record, result.error); } else if (command.type === "workspace.prune") { const result = await pruneWorktrees({ repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined, }); await relay.updateCommand(command.id, "succeeded", result); } else if (command.type === "workspace.unpublish-review") { // #1452 — REVIEW-ONLY unpublish of the worker tip's refs/review/* ref on origin (never // refs/heads/*/main, never lands). PUBLISH is intercepted earlier and runs the claim+fence+token- // gated path (runClaimGatedPublishReview); only unpublish reaches here. Safety is host-side. await handleReviewRefCommand(command, relay); } else if (command.type === "orchestrator.migrate-provider-config") { // Report our host-local provider config files; the relay seeds the central // provider-config rows with its own authority (#465). We don't write anything. await relay.updateCommand(command.id, "succeeded", { ...readLocalProviderConfigs() }); } else if (command.type === "orchestrator.upgrade") { // Install + restart ourselves. Intentionally NOT marked "succeeded": the // relay settles it by reconciling the version we report after we restart, // since the success ack can't survive our own process teardown. await handleSelfUpgrade(command, config, relay); } else { throw new Error(`unsupported orchestrator command: ${command.type}`); } await relay.updateManagedAgents(managedAgents); return true; } catch (error) { await relay.updateCommand(command.id, "failed", undefined, errMessage(error)); return false; } } function getManagedAgents(): ManagedAgentReport[] { return managedAgents; } function setManagedAgents(agents: ManagedAgentReport[]): void { managedAgents = agents; } return { handleCommand, getManagedAgents, setManagedAgents, spawnsInFlight: () => backgroundSpawns.size, settleSpawns }; } export function managedAgentShutdownTarget(agents: ManagedAgentReport[], ctrl: Record): ManagedAgentReport | undefined { // #1746 RC-1 — resolve the shutdown target by trying EVERY key present on the command in priority // order, returning the first that matches an agent. The prior code early-returned on the // first-SPECIFIED key (session) even when it did NOT match, so a session-name divergence stranded // the whole lookup at `undefined` — the force-kill machinery was never reached and the inert // shutdown was reported as `succeeded` while the process stayed alive (the silent zombie). A MISS on // one key must fall through to the next, never short-circuit the resolution. const requestedSession = typeof ctrl.sessionName === "string" && ctrl.sessionName ? ctrl.sessionName : typeof ctrl.tmuxSession === "string" && ctrl.tmuxSession ? ctrl.tmuxSession : undefined; if (requestedSession) { // #1583/#1746 RC-1(b) — the runner publishes tmux's RENDERED session name (`.`/`:`→`_`) while the // orchestrator record keeps the REQUESTED (dotted) name; normalize BOTH sides so a rendering // divergence can't miss. Raw equality is tried first, then the rendered comparison. const renderedRequest = renderTmuxSessionTarget(requestedSession); const bySession = agents.find((agent) => agent.sessionName === requestedSession || agent.tmuxSession === requestedSession || (agent.sessionName != null && renderTmuxSessionTarget(agent.sessionName) === renderedRequest) || renderTmuxSessionTarget(agent.tmuxSession) === renderedRequest); if (bySession) return bySession; } const agentId = typeof ctrl.agentId === "string" && ctrl.agentId ? ctrl.agentId : undefined; if (agentId) { const byAgentId = agents.find((agent) => agent.agentId === agentId); if (byAgentId) return byAgentId; } const spawnRequestId = typeof ctrl.spawnRequestId === "string" && ctrl.spawnRequestId ? ctrl.spawnRequestId : undefined; if (spawnRequestId) { const bySpawnRequestId = agents.find((agent) => agent.spawnRequestId === spawnRequestId); if (bySpawnRequestId) return bySpawnRequestId; } const policyName = typeof ctrl.policyName === "string" && ctrl.policyName ? ctrl.policyName : undefined; if (policyName) { const byPolicyName = agents.find((agent) => agent.policyName === policyName); if (byPolicyName) return byPolicyName; } return undefined; } // #1746 — headroom added to the orchestrator's graceful wait before it SIGKILLs the runner // HOST. The requested `timeoutMs` is the PROVIDER's graceful budget; the runner then needs a // beat more to run its own verified force-kill of the tmux session and exit. A Claude tmux // session is a detached daemon, so SIGKILL-ing the runner host does NOT take it down — if the // orchestrator kills the host before the runner finishes its reap, the session orphans (a // zombie). This margin lets the runner's own teardown win the race in the common case. const RUNNER_TEARDOWN_HEADROOM_MS = 8_000; function shutdownTimeoutMs(ctrl: Record): number | undefined { if (!(Number.isSafeInteger(ctrl.timeoutMs) && ctrl.timeoutMs > 0)) return undefined; return Math.min(ctrl.timeoutMs + RUNNER_TEARDOWN_HEADROOM_MS, 60_000); } function spawnOptionsFromRecord(source: Record, config: OrchestratorConfig): SpawnOptions { const fallbackProvider = config.providers[0] ?? (getAllManifests()[0]?.id ?? "claude"); const provider = typeof source.provider === "string" && getManifest(source.provider) ? source.provider : fallbackProvider; return { provider, cwd: source.cwd || config.baseDir, rig: typeof source.rig === "string" ? source.rig : undefined, model: modelFromControl(source), effort: typeof source.effort === "string" ? source.effort : undefined, profile: typeof source.profile === "string" ? source.profile : undefined, workspaceMode: normalizeWorkspaceMode(source.workspaceMode), lifecycle: normalizeAgentLifecycle(source.lifecycle) ?? "persistent", workspaceSymlinks: stringArray(source.workspaceSymlinks), agentProfile: isRecord(source.agentProfile) ? source.agentProfile : undefined, label: typeof source.label === "string" ? source.label : undefined, agentId: typeof source.agentId === "string" ? source.agentId : undefined, approvalMode: typeof source.approvalMode === "string" ? source.approvalMode : "guarded", prompt: typeof source.prompt === "string" ? source.prompt : undefined, systemPromptAppend: typeof source.systemPromptAppend === "string" ? source.systemPromptAppend : undefined, relayInjectionEvents: Array.isArray(source.relayInjectionEvents) ? source.relayInjectionEvents : undefined, tags: stringArray(source.tags), capabilities: stringArray(source.capabilities), providerArgs: stringArray(source.providerArgs), env: stringRecord(source.env), policyName: typeof source.policyName === "string" ? source.policyName : undefined, spawnRequestId: typeof source.spawnRequestId === "string" ? source.spawnRequestId : undefined, taskId: Number.isSafeInteger(source.taskId) && source.taskId > 0 ? source.taskId : undefined, automationId: typeof source.automationId === "string" ? source.automationId : undefined, automationRunId: typeof source.automationRunId === "string" ? source.automationRunId : undefined, requestedVia: typeof source.requestedVia === "string" ? source.requestedVia : undefined, resumeWorkspace: parseResumeWorkspace(source.resumeWorkspace), acquisition: parseProjectAcquisition(source.acquisition), }; } export const spawnOptionsFromControl = spawnOptionsFromRecord; export const spawnOptionsFromRestartSource = spawnOptionsFromRecord; function modelFromControl(ctrl: Record): string | undefined { return typeof ctrl.providerModel === "string" ? ctrl.providerModel : typeof ctrl.model === "string" ? ctrl.model : undefined; } function stringRecord(value: unknown): Record | undefined { if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; const entries = Object.entries(value).filter((entry): entry is [string, string] => typeof entry[1] === "string"); return entries.length ? Object.fromEntries(entries) : undefined; } function stringArray(value: unknown): string[] | undefined { return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : undefined; } function parseResumeWorkspace(value: unknown): import("./workspace-probe/types").ResumeWorkspaceTarget | undefined { if (!isRecord(value)) return undefined; const branch = typeof value.branch === "string" ? value.branch : undefined; const mode = value.mode === "attach" || value.mode === "branch-from" ? value.mode : undefined; if (!branch || !mode) return undefined; return { branch, mode, worktreePath: typeof value.worktreePath === "string" ? value.worktreePath : undefined, workspaceId: typeof value.workspaceId === "string" ? value.workspaceId : undefined, baseRef: typeof value.baseRef === "string" ? value.baseRef : undefined, baseSha: typeof value.baseSha === "string" ? value.baseSha : undefined, }; } function parseProjectAcquisition(value: unknown): import("./spawn/types").ProjectAcquisitionManifest | undefined { if (!isRecord(value)) return undefined; if (value.mode !== "project-root" || value.sync !== "ff-only") return undefined; const projectId = typeof value.projectId === "string" ? value.projectId : undefined; const rootPath = typeof value.rootPath === "string" ? value.rootPath : undefined; const cwd = typeof value.cwd === "string" ? value.cwd : undefined; const remoteUrl = typeof value.remoteUrl === "string" ? value.remoteUrl : undefined; if (!projectId || !rootPath || !cwd || !remoteUrl) return undefined; return { mode: "project-root", projectId, rootPath, cwd, remoteUrl, ref: typeof value.ref === "string" ? value.ref : undefined, sync: "ff-only", }; }