import { log, error as logError } from '../logger' import { storeCreateWorkItem, storeGetWorkItem, storeGetPendingWorkItem, storeUpdateWorkItem, storeListSessionsByEnvironment, storeGetEnvironment, storeListWorkItems, storeGetSession, } from '../store' import { config } from '../config' import { getBaseUrl } from '../config' import { touchSession, isSessionClosedStatus } from './session' import { touchEnvironmentActivity } from './environment' import type { WorkResponse } from '../types/api' /** Encode work secret as base64 JSON (no JWT — just API key as token) */ function encodeWorkSecret(): string { const payload = { version: 1, session_ingress_token: config.apiKeys[0] || '', api_base_url: getBaseUrl(), sources: [] as string[], auth: [] as string[], use_code_sessions: false, } return Buffer.from(JSON.stringify(payload)).toString('base64url') } export async function createWorkItem( environmentId: string, sessionId: string, overrides?: { spawnMode?: 'same-dir' | 'worktree' cwd?: string | null permissionMode?: string | null }, ): Promise { // Validate environment exists and is active const env = storeGetEnvironment(environmentId) if (!env) { throw new Error(`Environment ${environmentId} not found`) } if (env.status !== 'active') { throw new Error( `Environment ${environmentId} is not active (status: ${env.status})`, ) } // Work items are projections of their session. Every dispatch path — // initial create, reconnect, inactivity redispatch, attach — reads cwd / // spawn_mode / permission_mode from the session record, so a resumed // session can never silently fall back to the worker's startup directory. // Overrides exist only for the very first dispatch (before the session // write is visible) or for tests. const session = storeGetSession(sessionId) const cwd = overrides?.cwd ?? session?.cwd ?? env.directory ?? null const spawnMode = overrides?.spawnMode ?? session?.spawnMode ?? undefined const permissionMode = overrides?.permissionMode ?? session?.permissionMode ?? null const secret = encodeWorkSecret() const record = storeCreateWorkItem({ environmentId, sessionId, secret, spawnMode, cwd, permissionMode, }) log( `[RCS] Work item created: ${record.id} for env=${environmentId} session=${sessionId}` + `${spawnMode ? ` spawnMode=${spawnMode}` : ''}${cwd ? ` cwd=${cwd}` : ''}`, ) return record.id } /** * Dispatch an environment-scoped RPC through the work poll channel. * * The daemon polls work for session spawning; the same channel carries * non-session requests (today: directory listing for the new-session * dialog). The item has no session id — its `rpc.requestId` is what the * worker echoes when it POSTs the result. */ export async function createRpcWorkItem( environmentId: string, requestId: string, method: string, params: Record, ): Promise { const env = storeGetEnvironment(environmentId) if (!env) { throw new Error(`Environment ${environmentId} not found`) } if (env.status !== 'active') { throw new Error( `Environment ${environmentId} is not active (status: ${env.status})`, ) } const record = storeCreateWorkItem({ environmentId, sessionId: '', secret: encodeWorkSecret(), rpc: { requestId, method, params }, }) log( `[RCS] RPC work item created: ${record.id} env=${environmentId} rpc=${requestId} method=${method}`, ) return record.id } /** Long-poll for work — blocks until work is available or timeout. * Returns null when no work is available, matching the CLI bridge client protocol. * * Atomically transitions the item from `pending` → `dispatched` before * returning it, so that a subsequent poll (e.g. when ackWork failed or the * worker crashed mid-ack) does NOT re-return the same item immediately. * Stale `dispatched` items are reclaimed to `pending` by * `reclaimStaleDispatchedItems` after the lease TTL. `ackWork` transitions * `dispatched` → `acked`. */ export async function pollWork( environmentId: string, timeoutSeconds = config.pollTimeout, ): Promise { const deadline = Date.now() + timeoutSeconds * 1000 while (Date.now() < deadline) { reclaimStaleDispatchedItems(environmentId) const item = storeGetPendingWorkItem(environmentId) if (item) { // Atomically mark dispatched so the next poll doesn't re-return it. // If ackWork never arrives, reclaimStaleDispatchedItems will reset it. storeUpdateWorkItem(item.id, { state: 'dispatched' }) const base = { id: item.id, type: 'work' as const, environment_id: environmentId, state: 'dispatched', secret: item.secret, created_at: item.createdAt.toISOString(), } if (item.rpc) { return { ...base, data: { type: 'env_rpc', id: item.rpc.requestId, method: item.rpc.method, params: item.rpc.params, }, } } return { ...base, data: { type: 'session', id: item.sessionId, }, spawn_mode: item.spawnMode, cwd: item.cwd ?? undefined, permission_mode: item.permissionMode ?? undefined, } } await new Promise(r => setTimeout(r, 500)) } return null } /** Reset `dispatched` items older than the lease TTL back to `pending`. * Covers the case where a worker fetched an item but crashed before * acking — without this the item (and its session) is orphaned. */ function reclaimStaleDispatchedItems(environmentId: string): void { const ttlMs = config.heartbeatInterval * 2 * 1000 const now = Date.now() for (const item of storeListWorkItems()) { if ( item.environmentId === environmentId && item.state === 'dispatched' && now - item.updatedAt.getTime() > ttlMs ) { storeUpdateWorkItem(item.id, { state: 'pending' }) log( `[RCS] Reclaimed stale dispatched work item ${item.id} (session=${item.sessionId})`, ) } } } export function ackWork(workId: string) { storeUpdateWorkItem(workId, { state: 'acked' }) } export function stopWork(workId: string) { storeUpdateWorkItem(workId, { state: 'completed' }) } export function heartbeatWork(workId: string): { lease_extended: boolean state: string last_heartbeat: string ttl_seconds: number } { storeUpdateWorkItem(workId, {}) // just bump updatedAt const item = storeGetWorkItem(workId) // Also touch the associated session so the disconnect monitor doesn't // mark it inactive while the worker is still actively heartbeating. if (item?.sessionId) { touchSession(item.sessionId) } // Worker liveness for the parent env: an at-capacity worker (e.g. a // capacity=1 spawn-mode worker) stops polling for work for the whole // duration of a session (at-capacity poll interval is 10 min) and sends // only this heartbeat (~60s). Without counting it as env activity, // the disconnect monitor flips every busy env to 'disconnected' once // RCS_DISCONNECT_TIMEOUT (default 300s) passes since the dispatch poll. // Use the item's own environmentId — the route's :id param is not // validated against it. Completed (stopped → 410) items must NOT refresh // liveness: a zombie heartbeat after a Web-UI session deletion must not // resurrect the env. if (item && item.state !== 'completed') { touchEnvironmentActivity(item.environmentId) } const now = new Date() return { lease_extended: true, state: item?.state ?? 'acked', last_heartbeat: now.toISOString(), ttl_seconds: config.heartbeatInterval * 2, } } /** Reconnect: re-queue sessions associated with an environment */ export function reconnectWorkForEnvironment(envId: string) { const activeSessions = storeListSessionsByEnvironment(envId).filter( s => s.status === 'idle', ) const promises = activeSessions.map(s => createWorkItem(envId, s.id)) return Promise.all(promises) } /** * Server-initiated recovery dispatch for a single session. * * Fired when the session's ingress WS is closed for client inactivity * (ws-handler): with HybridTransport the inactivity clock only advances via * HTTP POST ingress (all writes go through POST — the WS carries no * app-level client→server frames), so 300s of POST silence with a live WS * means the worker's event uploader is wedged (e.g. one hung POST stalled * the serialized drain loop). The worker's WS auto-reconnects within * seconds — from the server's side everything looks healthy again, so * WITHOUT this re-dispatch the zombie transport is never replaced: the * at-capacity poll (10 min interval) returns no work and the worker's * write path stays dead forever (observed: session stuck 'running', all * worker events and control_responses silently swallowed). * * A fresh work item makes the worker's next poll fire onWorkReceived → * brand-new transport (fresh WS + fresh uploader) → recovery. Deliberately * NOT filtered by session status — a wedged worker's session is typically * stuck in 'running' (the turn-end result event was eaten), which is * exactly the case that needs the rebuild. */ export async function redispatchWorkForSession( sessionId: string, ): Promise { const session = storeGetSession(sessionId) if (!session) return if (isSessionClosedStatus(session.status)) return const envId = session.environmentId if (!envId) return // Don't stack items: if the env already has a pending or in-flight // dispatch for this cycle, the recovery is already queued. const hasOpenItem = storeListWorkItems().some( item => item.environmentId === envId && item.sessionId === sessionId && (item.state === 'pending' || item.state === 'dispatched'), ) if (hasOpenItem) return try { await createWorkItem(envId, sessionId) log( `[RCS] Redispatched work item after WS inactivity close: session=${sessionId} env=${envId}`, ) } catch (err) { // Env inactive (worker gone — disconnect monitor will handle it) or // race with deletion. Not an error worth surfacing. log( `[RCS] Redispatch after inactivity close skipped (session=${sessionId}): ${err instanceof Error ? err.message : String(err)}`, ) } }