import { fireAndForget, guardedInterval } from "./async-guard"; import type { OrchestratorConfig } from "./config"; import { tmuxSocketSweepEnabled, tmuxSocketSweepIntervalMs, tmuxSocketSweepProbeTimeoutMs, wedgedSessionIdleThresholdMs, wedgedSessionReapEnabled, wedgedSessionReapIntervalMs, } from "./config"; import { sweepStaleTmuxSockets } from "./tmux-socket-sweeper"; import { reapWedgedSessions } from "./wedged-session-reaper"; interface OrchestratorMaintenanceJobDefinition { id: string; title: string; enabled: boolean; intervalMs: number; runOnStart: boolean; handler(): unknown; } let started = false; const timers: Timer[] = []; export function maintenanceJobDefinitions(config: OrchestratorConfig): OrchestratorMaintenanceJobDefinition[] { return [ { id: "tmux-socket-sweep", title: "Tmux socket sweep", enabled: tmuxSocketSweepEnabled(), intervalMs: tmuxSocketSweepIntervalMs(), runOnStart: true, handler() { const result = sweepStaleTmuxSockets({ probeTimeoutMs: tmuxSocketSweepProbeTimeoutMs() }); if (result.scanned > 0 || result.failed.length > 0) { console.error( `[orchestrator] Tmux socket sweep: removed=${result.removed} kept=${result.kept} timedOut=${result.timedOut} scanned=${result.scanned} dir=${result.dir}`, ); } // #1760 — a socket whose liveness probe hit the timeout is a wedged/unresponsive tmux server. // Surface it distinctly: this is the host-level state that stalls the command loop, and a // plain restart won't clear it (the tmux server outlives the orchestrator). if (result.timedOut > 0) { console.error( `[orchestrator] Tmux socket sweep: ${result.timedOut} socket(s) unresponsive (list-sessions timed out at ${tmuxSocketSweepProbeTimeoutMs()}ms) — kept, not removed; a wedged tmux server may need manual intervention (#1760)`, ); } for (const failure of result.failed) { console.error(`[orchestrator] Tmux socket sweep failed for ${failure.socket}: ${failure.error}`); } return result; }, }, { // #1514 — reap orphaned, idle managed sessions (e.g. wedged at Claude's exit dialog). id: "wedged-session-reap", title: "Wedged session reap", enabled: wedgedSessionReapEnabled(), intervalMs: wedgedSessionReapIntervalMs(), runOnStart: true, handler() { const result = reapWedgedSessions({ tmuxPrefix: config.tmuxPrefix, idleThresholdMs: wedgedSessionIdleThresholdMs(), }); for (const session of result.reaped) { console.error( `[orchestrator] Reaped wedged session ${session.name} (unmanaged, idle ${Math.round(session.idleMs / 60_000)}m` + `${session.atExitDialog ? ", stuck at exit dialog" : ""})`, ); } if (result.sparedLiveOwner > 0) { console.error( `[orchestrator] Wedged session reap: spared ${result.sparedLiveOwner} unmanaged session(s) whose owning runner is still alive (positive-orphan backstop, #1514)`, ); } if (result.sparedKillRecheck > 0) { console.error( `[orchestrator] Wedged session reap: aborted ${result.sparedKillRecheck} kill(s) whose session identity/tracked state changed or could not be re-confirmed at kill time (identity-atomic kill, #1514 r8/r9)`, ); } if (result.enumerationIncomplete) { console.error( `[orchestrator] Wedged session reap: one or more tmux sockets failed to enumerate this cycle — tombstone sweep skipped, fleet-wide conclusions suppressed (#1514 r9)`, ); } return result; }, }, ]; } export function startOrchestratorMaintenanceScheduler(config: OrchestratorConfig): void { if (started) return; const definitions = maintenanceJobDefinitions(config).filter((definition) => definition.enabled); if (definitions.length === 0) return; started = true; for (const definition of definitions) { // #1676 — guarded even though runJob() catches internally: the guard is what makes // that a property of the SCHEDULER rather than of every job body, so a future edit // inside runJob cannot re-arm an unhandled rejection here. if (definition.runOnStart) void fireAndForget(`Maintenance job ${definition.id}`, () => runJob(definition)); timers.push(guardedInterval(`Maintenance job ${definition.id}`, definition.intervalMs, () => runJob(definition))); } } async function runJob(definition: OrchestratorMaintenanceJobDefinition): Promise { try { await Promise.resolve(definition.handler()); } catch (error) { console.error(`[orchestrator] Maintenance job ${definition.id} failed: ${error}`); } }