/** * Startup reconciliation for conversations left mid-turn by the previous * process. Their `processing_started_at` is still set even though the * in-memory agent loop that owned the turn died with that process, so the * flag is stale. * * Clearing the stale flags themselves runs out of process, in the monitor's * recovery pass (`monitoring/recovery/stale-processing.ts`), off the daemon's * boot path. This module owns the other half: when * `conversations.resumeProcessingOnStartup` is enabled it selects the * conversations to resume through the conversation-wake machinery — a normal * background turn that shows the model an interruption notice and lets it * decide what still needs doing, rather than mechanically replaying the dead * turn (tool side effects are not idempotent, and the transcript already * contains whatever the interrupted turn persisted). Selection reads the * still-set flags here at boot; the monitor clears them a few seconds later. * * Each resume runs under the conversation's reconstructed resting trust (see * `recoverRestingTrustContext` in `conversation-resting-trust.ts`); * conversations whose trust can't be rebuilt from persisted state are cleared * but left un-resumed. */ import { incrementProcessingResumeAttempts, listInterruptedConversations, } from "../persistence/conversation-crud.js"; import { getLogger } from "../util/logger.js"; import { recoverRestingTrustContext } from "./conversation-resting-trust.js"; import type { TrustContext } from "./trust-context-types.js"; const log = getLogger("interrupted-turns"); /** * Maximum consecutive auto-resume attempts per conversation. The persisted * counter survives the stale-flag clear and resets only on a clean turn end, * so a resumed turn that keeps taking the process down is left idle after this * many boots instead of resume-looping forever. */ export const MAX_RESUME_ATTEMPTS = 2; /** * Wake hint injected into the resumed turn. Deliberately instructs the model * to judge what remains unfinished instead of redoing the turn wholesale — * effects of already-executed tools are visible in the transcript and must * not be repeated. */ const INTERRUPTED_TURN_RESUME_HINT = "Your previous turn in this conversation was interrupted by an assistant " + "restart before it finished. Review the recent messages and complete " + "whatever was left unfinished. Do not repeat actions whose effects are " + "already visible in the conversation. If nothing actionable remains, send " + "a brief note acknowledging the reply was cut short so the conversation " + "isn't left hanging."; /** * A conversation selected for an auto-resume wake, paired with the resting * trust context the woken turn must run under. */ export interface InterruptedResumeTarget { conversationId: string; trustContext: TrustContext; } export interface InterruptedTurnReconciliation { /** Conversations selected for an auto-resume wake, with their resting trust. */ resume: InterruptedResumeTarget[]; /** Conversation ids left idle because they hit {@link MAX_RESUME_ATTEMPTS}. */ capped: string[]; /** * Conversation ids left un-resumed because their resting trust could not be * reconstructed from persisted state: a remote-channel turn whose per-actor * gateway verdict is not stored, or an origin the build does not recognize. * Their stale flag is still cleared. */ trustUnrecoverable: string[]; } /** * When `resumeEnabled` is set, pick the conversations to resume together with * the resting trust each wake runs under. Conversations whose resting trust * can't be recovered are skipped. The resume-attempt counter is NOT bumped * here — it is charged as each wake starts (see {@link * resumeInterruptedConversations}), so a crash mid-resume never burns the * budget of conversations that were never attempted. * * Reads the still-set `processing_started_at` flags (the monitor's recovery * pass clears them out of process). Pure read — safe to run during startup as * soon as migrations settle. The wakes themselves must wait for full startup * (providers, CES) and run via {@link resumeInterruptedConversations}. */ export function reconcileInterruptedConversations( resumeEnabled: boolean, ): InterruptedTurnReconciliation { if (!resumeEnabled) { return { resume: [], capped: [], trustUnrecoverable: [] }; } const interrupted = listInterruptedConversations(); const resume: InterruptedResumeTarget[] = []; const capped: string[] = []; const trustUnrecoverable: string[] = []; for (const row of interrupted) { if (row.resumeAttempts >= MAX_RESUME_ATTEMPTS) { capped.push(row.id); continue; } const trustContext = recoverRestingTrustContext(row.id); if (!trustContext) { trustUnrecoverable.push(row.id); continue; } resume.push({ conversationId: row.id, trustContext }); } return { resume, capped, trustUnrecoverable }; } /** * Resume interrupted conversations sequentially through the conversation * wake machinery. Sequential on purpose: a boot after a mid-turn crash can * carry several interrupted conversations, and running them one at a time * avoids a thundering herd of concurrent LLM turns right after startup. * * The persisted resume-attempt counter is bumped immediately before each * conversation's wake — never up-front for the whole batch — so a resume that * takes the process down again only charges the conversation actually being * attempted, leaving the rest of the budget intact across the next boots. * * Each wake runs clientless (background policy: side-effecting tools are * denied at the default threshold instead of stalling on an absent client) * under the conversation's reconstructed resting trust ({@link * InterruptedResumeTarget}), so resuming a conversation can never grant it more * capability than its own resting trust. Failures are logged and skipped so * one bad conversation doesn't block the rest. * * `agent-wake` is imported lazily: this module is loaded during early daemon * startup, and the wake machinery statically pulls in the agent-loop stack, * which must not join the lifecycle import graph (daemon ↔ runtime cycles). */ export async function resumeInterruptedConversations( targets: InterruptedResumeTarget[], ): Promise { const { wakeAgentForOpportunity } = await import("../runtime/agent-wake.js"); for (const { conversationId, trustContext } of targets) { try { // Charge the attempt before the wake runs. The cap exists to stop a // resumed turn that keeps killing the process, so the counter must be // durably incremented before that turn can crash the daemon; charging it // per-conversation (never up-front for the batch) also means a crash // mid-resume never burns the budget of conversations still queued behind // it. Doing it inside this guarded block keeps a transient counter-write // failure scoped to its own conversation instead of aborting every // resume still queued. incrementProcessingResumeAttempts(conversationId); const result = await wakeAgentForOpportunity({ conversationId, hint: INTERRUPTED_TURN_RESUME_HINT, source: "interrupted-turn-resume", trustContext, clientless: true, persistTriggerAsEvent: true, }); log.info( { conversationId, invoked: result.invoked, producedToolCalls: result.producedToolCalls, reason: result.reason, }, "Interrupted-turn resume wake finished", ); } catch (err) { log.warn( { err, conversationId }, "Interrupted-turn resume wake failed — continuing with remaining conversations", ); } } }