import { log } from '../logger' import { storeListSessions } from '../store' import { config } from '../config' import type { SessionEvent } from '../transport/event-bus' import { getEventBus } from '../transport/event-bus' import { updateSessionStatus } from './session' import { redispatchWorkForSession } from './work-dispatch' /** * Stuck-turn watchdog — the server-side self-heal for sessions whose * turn-end signal was lost in transit. * * A session's status only returns from `running` to `idle` when the worker's * `result`/`result_success` event arrives (ws-handler.ts ingestBridgeMessage) * or the user interrupts. The worker sends that result fire-and-forget — * when the write is dropped (transport-swap window, uploader batch drop * after maxConsecutiveFailures, close-grace expiry), it is never re-sent and * the session stays `running` forever. The Web UI locks the input (black * stop square) and every new message lands in a queue that never drains. * * Nothing else detects this state: * - the worker's write-path watchdog only fires when POSTs stop completing * entirely (a transient drop followed by healthy POSTs is invisible); * - the WS inactivity clock is kept fresh by keepalive POSTs; * - the control-request watchdog only tracks RPCs the server itself sent. * * The sweep flips a `running` session with no turn signal for * `config.stuckTurnTimeout` back to `idle` (updateSessionStatus publishes * `session_status`, unlocking live SSE viewers immediately — a genuinely * still-running turn re-derives `running` on its next event) and fires the * recovery work re-dispatch (zombie-transport rebuild, same as the WS * inactivity close). * * `requires_action` is deliberately out of scope: a permission ask that * arrived is visible and actionable in the Web UI — the user is the one * blocking, not a lost signal. */ /** * Inbound event types that count as turn progress — an ALLOWLIST, so future * chatter types default to not feeding the signal clock: * - `user` (worker-mirrored tool results), `assistant`, `result`/ * `result_success`, `error` — the conversation itself; * - `system` — subagent task events (task_started/task_progress/…). * Exception: subtype `init` is re-sent on every transport (re)connect — * transport liveness, not turn progress. * * Deliberately NOT counted: * - `control_request`/`control_response` — RPC traffic. The Web UI polls * get_context_usage every 60s while the page is open, so the worker's * control_response answers flow constantly even for a wedged session — * counting them would make the watchdog permanently blind exactly when * the user is watching the stuck session. * - `models_state`, `system/init` — connect-time snapshots. * - `task_state` — background task-list churn; a background task running * outside any turn must not starve the watchdog. * - `session_status` — server-synthesized (including this watchdog's own * idle flip; counting it would make the sweep single-shot). */ const SIGNAL_EVENT_TYPES = new Set([ 'user', 'assistant', 'partial_assistant', 'result', 'result_success', 'error', 'system', ]) function isInboundSignal(event: SessionEvent): boolean { if (event.direction !== 'inbound') return false if (!SIGNAL_EVENT_TYPES.has(event.type)) return false if (event.type === 'system') { // system/init is a connect-time snapshot, re-sent on every // (re)connect — not turn progress. const subtype = (event.payload as Record | null)?.subtype if (subtype === 'init') return false } return true } /** * Wall-clock of the latest event on the bus that proves the session's turn * is (or was) progressing: any inbound worker conversation event (assistant, * tool-result user, result, subagent task events), or the outbound user * message that started the turn. Returns 0 when the bus carries no signal * at all (fresh bus after a server restart — callers fall back to the * persisted session record). */ export function lastTurnSignalAt(events: SessionEvent[]): number { for (let i = events.length - 1; i >= 0; i--) { const event = events[i] if (isInboundSignal(event)) { return event.createdAt } if (event.direction === 'outbound' && event.type === 'user') { return event.createdAt } } return 0 } /** * Sweep all sessions for stuck turns. Returns the number of sessions healed. * `now` is injectable for tests. */ export function sweepStuckTurnSessions(now: number = Date.now()): number { const thresholdMs = config.stuckTurnTimeout * 1000 if (!(thresholdMs > 0)) return 0 let swept = 0 for (const session of storeListSessions()) { // Only `running` locks the input with no recovery path; see module docs // for why requires_action (visible, actionable card) and archived // sessions are excluded. if (session.status !== 'running') continue // ACP sessions carry their turn traffic on the per-channel-group ACP // bus (getAcpEventBus), not the session bus this sweep scans — healing // them from the wrong bus would be a false positive. if (session.source === 'acp') continue const bus = getEventBus(session.id) const lastSignal = lastTurnSignalAt(bus.getEventsSince(0)) // Empty bus (server restart) or a signal older than the session record // (persisted legacy) — fall back to the persisted creation time so // restart-legacy stuck sessions still heal after the same grace window. const reference = Math.max(lastSignal, session.createdAt.getTime()) if (now - reference <= thresholdMs) continue const silenceSec = Math.round((now - reference) / 1000) log( `[RCS] Stuck-turn watchdog: session=${session.id} running with no turn signal for ${silenceSec}s` + ' — flipping to idle (turn-end result was lost) and redispatching work', ) // Recovery re-dispatch first: when the worker's transport is actually // dead this rebuilds it (fresh uploader → the write path recovers). // No-op for healthy transports with an unchanged token. void redispatchWorkForSession(session.id) // Publishes `session_status` → live SSE viewers unlock the input. updateSessionStatus(session.id, 'idle') swept++ } return swept }