/** * Regression: `slz rcs one` session stuck on `running` forever after the * worker's turn-end result event is lost in transit. * * Reproduction of the reported symptom: task half-done, a tool call succeeds * (its tool-result user event reaches the server), then silence — the web * input shows the black stop square (isLoading) and every new message lands * in the queue forever. * * Root cause chain (worker side, see src/bridge/outboundRetry.ts + * replBridge.ts): sendResult()/sendControlRequest() are fire-and-forget — * a result written during a transport swap window ("DROP sendResult: no * transport") or dropped by the uploader (maxConsecutiveFailures / close * grace) is never re-sent. The server-side status derivation only returns * to idle on an inbound result, so the session stays `running` forever and * NOTHING self-heals it: the worker write watchdog is quiet (POSTs are * healthy again), the WS inactivity clock is kept fresh by keepalive POSTs, * and the control-request watchdog only tracks RPCs the server itself sent. * * This test pins the server-side self-heal: the stuck-turn watchdog sweep * flips stale `running` sessions back to idle (publishing session_status so * live SSE viewers unlock the input) and fires the recovery work re-dispatch. * `requires_action` is deliberately out of scope — a visible permission card * is actionable, not stuck. */ import { describe, test, expect, beforeEach, mock } from 'bun:test' import { mockConfigModule } from './helpers/mock-config' mock.module('../config', () => mockConfigModule({ pollTimeout: 1, stuckTurnTimeout: 60 }), ) import { Hono } from 'hono' import { storeReset, storeCreateEnvironment, storeGetPendingWorkItem, } from '../store' import { getAllEventBuses, removeEventBus, getEventBus, type SessionEvent, } from '../transport/event-bus' import v1SessionIngress from '../routes/v1/session-ingress' import webSessions from '../routes/web/sessions' import webControl from '../routes/web/control' import { issueToken } from '../auth/token' import { getSession } from '../services/session' import { sweepStuckTurnSessions } from '../services/stuck-turn-watchdog' /** Matches the mocked config's stuckTurnTimeout (60s). */ const THRESHOLD_MS = 60 * 1000 function createApp() { const app = new Hono() app.route('/v2/session_ingress', v1SessionIngress) app.route('/web', webSessions) app.route('/web', webControl) return app } const AUTH_HEADERS = { Authorization: 'Bearer test-api-key', 'X-Username': 'testuser', } function auth(user: string): string { return `Bearer ${issueToken(user).token}` } /** Drive the session exactly like the real worker does: the Web UI starts * the turn (outbound user), HybridTransport POSTs the worker's events via * session-ingress (inbound assistant / tool-result user). */ async function driveTurn(app: Hono, sessionId: string): Promise { const sendRes = await app.request(`/web/sessions/${sessionId}/events`, { method: 'POST', headers: { Authorization: auth('user-1'), 'Content-Type': 'application/json', }, body: JSON.stringify({ type: 'user', uuid: 'web-uuid-1', content: 'run the task', message: { content: 'run the task' }, }), }) expect(sendRes.status).toBe(200) // Worker: assistant streams (running), then the tool executes successfully // and its tool-result user message is mirrored — the last events the // server ever sees. const ingestRes = await app.request( `/v2/session_ingress/session/${sessionId}/events`, { method: 'POST', headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' }, body: JSON.stringify({ events: [ { type: 'assistant', uuid: 'asst-uuid-1', message: { role: 'assistant', content: [ { type: 'tool_use', id: 't1', name: 'Bash', input: {} }, ], }, }, { type: 'user', uuid: 'tool-result-1', message: { role: 'user', content: [ { type: 'tool_result', tool_use_id: 't1', content: 'ok' }, ], }, }, ], }), }, ) expect(ingestRes.status).toBe(200) } describe('stuck-turn watchdog (lost turn-end result regression)', () => { let app: Hono let sessionId: string let envId: string beforeEach(async () => { storeReset() for (const [key] of getAllEventBuses()) { removeEventBus(key) } app = createApp() // Owned by the caller: Web session creation now enforces machine-level // access (owner / team / admin). const env = storeCreateEnvironment({ secret: 's', ownerUserId: 'user-1' }) envId = env.id const res = await app.request('/web/sessions', { method: 'POST', headers: { Authorization: auth('user-1'), 'Content-Type': 'application/json', }, body: JSON.stringify({ environment_id: envId }), }) const body = (await res.json()) as { id: string } sessionId = body.id }) test('lost turn-end result: session stuck running, sweep heals to idle', async () => { await driveTurn(app, sessionId) // ── The reproduction: the turn-end result event never arrives ── // (dropped by the worker's uploader / transport-swap window). The // session is now stuck on `running` — the web input locks and every // new message queues forever. expect(getSession(sessionId)?.status).toBe('running') // Time passes; the worker's keepalive POSTs keep the WS alive but no // conversation events arrive. Nothing on the server heals the status… const healed = sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000) expect(healed).toBe(1) expect(getSession(sessionId)?.status).toBe('idle') // A session_status event must be broadcast so live SSE viewers // (RCSChatAdapter onStatusChange) unlock the input immediately. const statuses = getEventBus(sessionId) .getEventsSince(0) .filter(e => e.type === 'session_status') .map(e => (e.payload as Record).status) expect(statuses[statuses.length - 1]).toBe('idle') // The sweep also queues the recovery work re-dispatch (zombie-transport // rebuild) — same recovery the WS inactivity close uses. expect(storeGetPendingWorkItem(envId)?.sessionId).toBe(sessionId) }) test('genuinely progressing turn is NOT healed (recent inbound events)', async () => { await driveTurn(app, sessionId) // Worker still streaming — the sweep with fresh clock must not flip. expect(sweepStuckTurnSessions()).toBe(0) expect(getSession(sessionId)?.status).toBe('running') // A fresh inbound assistant resets the silence clock — sweeping before // the full threshold elapses since that event must not heal. const ingestRes = await app.request( `/v2/session_ingress/session/${sessionId}/events`, { method: 'POST', headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' }, body: JSON.stringify({ events: [ { type: 'assistant', uuid: 'asst-uuid-2', message: { role: 'assistant', content: [{ type: 'text', text: 'still working' }], }, }, ], }), }, ) expect(ingestRes.status).toBe(200) // The assistant arrived moments ago — sweeping at threshold-1s (i.e. the // last signal is younger than the threshold) must not heal. expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0) expect(getSession(sessionId)?.status).toBe('running') }) test('turn just started (outbound user, no worker reply yet) is NOT healed', async () => { const res = await app.request(`/web/sessions/${sessionId}/events`, { method: 'POST', headers: { Authorization: auth('user-1'), 'Content-Type': 'application/json', }, body: JSON.stringify({ type: 'user', uuid: 'fresh-user-1', content: 'hello', message: { content: 'hello' }, }), }) expect(res.status).toBe(200) expect(getSession(sessionId)?.status).toBe('running') // The outbound user event marks the turn start — the worker gets the // full threshold to produce its first inbound event. expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0) expect(getSession(sessionId)?.status).toBe('running') // …but a turn whose prompt is older than the threshold with zero worker // output (worker never picked it up) does heal. expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1) expect(getSession(sessionId)?.status).toBe('idle') }) test('60s control RPC answers do NOT count as turn progress', async () => { await driveTurn(app, sessionId) // The Web UI polls get_context_usage every 60s while the page is open — // the worker's control_response answers are RPC liveness, not turn // progress. They must not keep a wedged turn looking alive. const pollRes = await app.request(`/web/sessions/${sessionId}/control`, { method: 'POST', headers: { Authorization: auth('user-1'), 'Content-Type': 'application/json', }, body: JSON.stringify({ type: 'control_request', request_id: 'rpc-1', request: { subtype: 'get_context_usage' }, }), }) expect(pollRes.status).toBe(200) const answerRes = await app.request( `/v2/session_ingress/session/${sessionId}/events`, { method: 'POST', headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' }, body: JSON.stringify({ events: [ { type: 'control_response', response: { subtype: 'success', request_id: 'rpc-1', response: {}, }, }, ], }), }, ) expect(answerRes.status).toBe(200) expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1) expect(getSession(sessionId)?.status).toBe('idle') }) test('connect-time snapshots and background task_state do not feed the signal clock', async () => { await driveTurn(app, sessionId) // system/init (re-ssent on every reconnect), models_state, and // task_state (background task churn outside any turn) are transport / // task-list liveness, not turn progress — a wedged session must still // heal even while these keep arriving. const chatterRes = await app.request( `/v2/session_ingress/session/${sessionId}/events`, { method: 'POST', headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' }, body: JSON.stringify({ events: [ { type: 'system', subtype: 'init', uuid: 'init-1', message: {}, }, { type: 'models_state', uuid: 'models-1', message: {}, }, { type: 'task_state', uuid: 'tasks-1', message: {}, }, ], }), }, ) expect(chatterRes.status).toBe(200) expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1) expect(getSession(sessionId)?.status).toBe('idle') }) test('subagent task events DO feed the signal clock (genuine turn progress)', async () => { await driveTurn(app, sessionId) const progressRes = await app.request( `/v2/session_ingress/session/${sessionId}/events`, { method: 'POST', headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' }, body: JSON.stringify({ events: [ { type: 'system', subtype: 'task_progress', uuid: 'task-progress-1', task_id: 'task-1', message: {}, }, ], }), }, ) expect(progressRes.status).toBe(200) // The task_progress event just arrived — sweeping before the full // threshold since it must not heal. expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0) expect(getSession(sessionId)?.status).toBe('running') }) test('pending permission ask (requires_action) is out of scope — never healed', async () => { await driveTurn(app, sessionId) // The worker's next tool needs permission — the can_use_tool ask is // visible and actionable in the Web UI (card with approve/deny), so a // requires_action session is NOT stuck: the user is the one blocking. // The sweep must never touch it, no matter how long the user waits. const askRes = await app.request( `/v2/session_ingress/session/${sessionId}/events`, { method: 'POST', headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' }, body: JSON.stringify({ events: [ { type: 'control_request', request_id: 'perm-1', request: { subtype: 'can_use_tool', tool_name: 'Bash', input: {}, tool_use_id: 't2', }, }, ], }), }, ) expect(askRes.status).toBe(200) expect(getSession(sessionId)?.status).toBe('requires_action') expect(sweepStuckTurnSessions(Date.now() + 10 * THRESHOLD_MS)).toBe(0) expect(getSession(sessionId)?.status).toBe('requires_action') }) test('lost permission ask (stuck on running, card never shown) heals', async () => { await driveTurn(app, sessionId) // The ask itself was lost in transit — status never left running and no // card is shown. Same sweep heals it (input unlock; the worker-side // retry re-delivers the ask and flips it to requires_action). expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1) expect(getSession(sessionId)?.status).toBe('idle') }) test('idle sessions and fresh buses are untouched', async () => { await driveTurn(app, sessionId) sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000) expect(getSession(sessionId)?.status).toBe('idle') // Already idle — the sweep must not publish anything new. const before = getEventBus(sessionId) .getEventsSince(0) .filter(e => e.type === 'session_status').length expect(sweepStuckTurnSessions(Date.now() + 10 * THRESHOLD_MS)).toBe(0) const after = getEventBus(sessionId) .getEventsSince(0) .filter(e => e.type === 'session_status').length expect(after).toBe(before) }) test('worker exit signal (synthetic error result) unlocks the session', async () => { await driveTurn(app, sessionId) expect(getSession(sessionId)?.status).toBe('running') // The bridge worker's session-exit signal: the `--print` child died // mid-turn (crash / timeout kill) and bridgeMain posts a synthetic // result straight to the session-ingress events endpoint. The server // must derive idle AND preserve the reason text for the Web UI. const exitRes = await app.request( `/v2/session_ingress/session/${sessionId}/events`, { method: 'POST', headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' }, body: JSON.stringify({ events: [ { type: 'result', subtype: 'error_during_execution', is_error: true, result: 'Worker process exited: OOM', duration_ms: 1234, num_turns: 0, uuid: 'exit-result-1', session_id: sessionId, }, ], }), }, ) expect(exitRes.status).toBe(200) expect(getSession(sessionId)?.status).toBe('idle') // Reason text + is_error must survive normalizePayload so the web // adapter can render the error card (not just silently unlock). const exitEvent = getEventBus(sessionId) .getEventsSince(0) .find(e => e.type === 'result') const payload = exitEvent?.payload as Record expect(payload.result).toBe('Worker process exited: OOM') expect(payload.is_error).toBe(true) expect(payload.subtype).toBe('error_during_execution') }) test('server-restart legacy: running session with an empty bus heals', async () => { // A session persisted as `running` whose bus is fresh (server restart) // has no turn signal at all — the sweep falls back to the session's // creation time so restart-legacy stuck sessions still heal. const { storeUpdateSession } = await import('../store') storeUpdateSession(sessionId, { status: 'running' }) removeEventBus(sessionId) // simulate fresh bus after restart const swept = sweepStuckTurnSessions(Date.now() + 1000) // Session was created milliseconds ago — creation-time fallback gives // the same grace window as a fresh turn. expect(swept).toBe(0) expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1) expect(getSession(sessionId)?.status).toBe('idle') }) test("watchdog's own session_status event does not count as turn progress", async () => { await driveTurn(app, sessionId) sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000) expect(getSession(sessionId)?.status).toBe('idle') // Flip it back to running (e.g. a late assistant with turn signal) and // verify the sweep's previously published session_status (inbound) did // not arm the signal clock — a sweep right past the threshold still // fires, and repeated sweeps keep healing. const { storeUpdateSession } = await import('../store') storeUpdateSession(sessionId, { status: 'running' }) expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 10_000)).toBe(1) expect(getSession(sessionId)?.status).toBe('idle') }) })