import { type Agent, AgentBusyError, type AgentMessage, isSyntheticToolResultMessage, type ThinkingLevel, } from "@oh-my-pi/pi-agent-core"; import type { AssistantMessage, AssistantRetryRecovery, AssistantRetryRecoveryKind, CodexCompactionContext, Effort, Model, ModelUsageHealth, TextContent, ToolChoice, AnthropicFallbackCreditHandle, } from "@oh-my-pi/pi-ai"; import { calculateRateLimitBackoffMs, isBillingCapUsageLimit, parseRateLimitReason } from "@oh-my-pi/pi-ai"; import * as AIError from "@oh-my-pi/pi-ai/error"; import { shouldDropTruncatedThinkingOnlyAssistant } from "@oh-my-pi/pi-ai/providers/transform-messages"; import { extractProviderRetryHint } from "@oh-my-pi/pi-ai/utils/retry-after"; import { fallbackCreditTargets } from "@oh-my-pi/pi-catalog/compat/fallback-credit"; import { resolveModelPolicy } from "@oh-my-pi/pi-catalog/compat/resolve"; import { isFireworksFastModelId, toFireworksBaseModelId } from "@oh-my-pi/pi-catalog/fireworks-model-id"; import { modelsAreEqual } from "@oh-my-pi/pi-catalog/models"; import { isUnexpectedSocketCloseMessage, logger, prompt, sleepLong, sleepUntil } from "@oh-my-pi/pi-utils"; import type { ModelRegistry } from "../config/model-registry"; import { formatModelStringWithRouting, resolveModelOverride } from "../config/model-resolver"; import type { Settings } from "../config/settings"; import type { RetryErrorUpdate } from "../extensibility/shared-events"; import { isEvalStateContext } from "../eval/state"; import emptyStopRetryTemplate from "../prompts/system/empty-stop-retry.md" with { type: "text" }; import malformedFunctionCallRetryTemplate from "../prompts/system/malformed-function-call-retry.md" with { type: "text" }; import streamStallContinueTemplate from "../prompts/system/stream-stall-continue.md" with { type: "text" }; import thinkingLoopRedirectTemplate from "../prompts/system/thinking-loop-redirect.md" with { type: "text" }; import unexpectedStopRetryTemplate from "../prompts/system/unexpected-stop-retry.md" with { type: "text" }; import { AUTO_THINKING, type ConfiguredThinkingLevel, clampThinkingLevelToCeiling, modelSupportsEffortCeiling, resolveThinkingLevelForModel, } from "@oh-my-pi/pi-tui/thinking"; import type { EditModeSetting } from "../edit/settings"; import type { AgentSessionEvent } from "./agent-session-events"; import type { ResetRecoveryResult } from "./codex-auto-reset"; import type { InitialRetryFallbackState, UsageFallbackConfirmation, UsageFallbackConfirmer, } from "./agent-session-types"; import { assistantTurnProducedOutput, isEmptyAssistantStop, isEmptyErrorTurn, isUserAuthoredMessage, operatorTitleText, } from "./messages"; import { type ActiveRetryFallbackState, calculateRetryBackoffDelayMs, findRetryFallbackCandidates, formatRetryFallbackBaseSelector, formatRetryFallbackSelector, getRetryFallbackChains, getRetryFallbackRevertPolicy, parseRetryFallbackSelector, type RetryFallbackChains, type RetryFallbackResolutionContext, type RetryFallbackRevertPolicy, type RetryFallbackSelector, resolveRetryFallbackChainKey, retryFallbackAdmits, type ServingModel, validateRetryFallbackChains, } from "./retry-fallback-chains"; import { describeUsageFallback } from "./retry-fallback-reason"; import { getLatestCompactionEntry } from "./session-context"; import { EPHEMERAL_MODEL_CHANGE_ROLE, type SessionEntry } from "./session-entries"; import type { SessionManager } from "./session-manager"; import { isUserQueuedMessage } from "./queued-messages"; import { sameMessageContent, sessionMessagePersistenceKey } from "./turn-persistence"; import { journalJudgmentUsage } from "../judgment"; import { classifyUnexpectedStop, isUnexpectedStopCandidate } from "./unexpected-stop-classifier"; import { cfgFeaturesUnexpectedStopDetection, cfgModelLoopGuardEnabled, cfgRetry, cfgRetryEnabled, cfgRetryModelFallback, cfgRetryUsageAwareFallback, cfgRetryUsageReservePct, cfgRetryUsageReservePolicy, } from "./settings"; const THINKING_LOOP_REDIRECT_TYPE = "thinking-loop-redirect"; const UNEXPECTED_STOP_MAX_RETRIES = 3; // Gateway-routed judges can take ~12s (observed: bifrost judge YES at 12.2s // while the client aborted at 4s, dropping the verdict and skipping the nudge). // Matches the 15s judgment-adjacent budgets (auth-gateway strict probe, // auto-graph pick) so a slow-but-healthy verdict still lands. const UNEXPECTED_STOP_TIMEOUT_MS = 15_000; const EMPTY_STOP_MAX_RETRIES = 3; const MALFORMED_FUNCTION_CALL_MAX_RETRIES = 3; const STREAM_STALL_CONTINUE_MAX_RETRIES = 3; const GENERIC_ABORT_MAX_RETRIES = 2; const SIBLING_UNBLOCK_BUFFER_MS = 1_000; const NON_WHITESPACE_RE = /\S/; const USAGE_PREFLIGHT_BLOCKED_PREFIX = "Usage preflight blocked:"; const STREAM_STALL_ERROR_RE = /stream stall/i; const HTTP2_STREAM_RESET_ERROR_RE = /stream closed with error code\s+nghttp2_(?:internal_error|refused_stream)|nghttp2_(?:internal_error|refused_stream)|HTTP2(?:StreamReset|RefusedStream)/i; // Gateway/provider closes a stream mid-generation without its terminal chunk // (openai-completions "finish_reason", openai/azure responses "terminal // response event", Codex "terminal completion event", Cursor "turnEnded" — // an HTTP/2 reset can settle as the stream end rather than the RST error). // Same transport-failure class as the stall/reset entries: retriable, and // eligible for preserved-turn continuation on resolved tool turns. const PREMATURE_STREAM_CLOSE_ERROR_RE = /(?:stream closed before a (?:finish_reason|terminal response event)|Codex stream ended before terminal completion event|Cursor stream ended before turnEnded)/i; /** * Bare abort sentinels with no provider reason: our own `"Request was aborted"` * plus Node/Bun `AbortError`s (`"The operation was aborted"`) surfaced by * transports when an internal (non-caller) abort fires. */ const GENERIC_ABORT_MESSAGES: Record = { "Request was aborted": true, "Request was aborted.": true, "The operation was aborted": true, "The operation was aborted.": true, }; function hasNonWhitespace(value: string): boolean { return NON_WHITESPACE_RE.test(value); } /** * End of the conversation in `messages`, before trailing context-only custom * messages: eval runtime snapshots (appended at startup and on display-context * rebuilds) and deliveries folded in without a turn — e.g. an `async-result` * landing while a user interrupt suppresses auto-resume. They trail a failed * turn without ending it. A user-authored custom prompt is conversation. */ function conversationEnd(messages: readonly AgentMessage[]): number { let end = messages.length; while (end > 0 && messages[end - 1].role === "custom" && !isUserAuthoredMessage(messages[end - 1])) end--; return end; } function syntheticToolResultTailStart(messages: readonly AgentMessage[]): number { let index = conversationEnd(messages); while (index > 0 && isSyntheticToolResultMessage(messages[index - 1])) { index--; } return index; } function retryableAssistantTurnEnd(messages: readonly AgentMessage[]): number | undefined { const turnEnd = syntheticToolResultTailStart(messages); const message = messages[turnEnd - 1]; if (message?.role !== "assistant") return undefined; if (message.stopReason !== "error" && message.stopReason !== "aborted") return undefined; return turnEnd; } /** An assistant turn cut off before any text or tool call; no provider replays it. */ function isCutOffAssistant(message: AgentMessage): boolean { return message.role === "assistant" && shouldDropTruncatedThinkingOnlyAssistant(message); } /** * The last prompt in `messages[0, end)` that no replayed assistant turn has * answered yet: a user prompt, or the operator's `.`/`c` continue directive. */ function unansweredPrompt(messages: readonly AgentMessage[], end: number): AgentMessage | undefined { for (let index = end - 1; index >= 0; index--) { const message = messages[index]; if (message.role === "assistant") { if (!isCutOffAssistant(message)) return undefined; } else if (isUserQueuedMessage(message) || (message.role === "developer" && message.userInitiated === true)) { return message; } } return undefined; } /** Whether `messages` ends in a failed/aborted turn whose most recent activity was a tool call. */ function abortedToolCallTail(messages: readonly AgentMessage[]): boolean { const turnEnd = retryableAssistantTurnEnd(messages); if (turnEnd === undefined) return false; // Synthetic tool results only ever trail a turn that emitted tool calls. if (isSyntheticToolResultMessage(messages[turnEnd])) return true; const beforeTurn = messages[turnEnd - 2]; return beforeTurn?.role === "toolResult" && beforeTurn.isError === true; } /** * Index of the first trailing failed tool result to strip for a tool replay, * or undefined when the tail does not qualify. * * Qualifies when the transcript ends in a fully failed/aborted tool batch * whose anchor assistant turn completed with runnable arguments * (`toolUse`/`stop`) — optionally followed by an aborted boundary with no tool * calls of its own (a live Esc; a restored session has already dropped it). * Stripping from the returned index leaves that assistant as the transcript * tail, which `Agent.continue()` resumes by re-executing the calls. A * partially successful batch never qualifies: re-running a completed call * would repeat its side effects. Synthetic placeholder tails never qualify * either — their anchor is the aborted turn itself and its arguments may be * truncated, so those are re-generated via a fresh model call instead. Nor * does a tail carrying a delivery other than an eval snapshot: the strip * would erase it from history, while a fresh model call keeps it in context. */ function toolReplayStart(messages: readonly AgentMessage[]): number | undefined { let end = conversationEnd(messages); for (let i = end; i < messages.length; i++) { if (!isEvalStateContext(messages[i])) return undefined; } const tail = messages[end - 1]; if (tail?.role === "assistant") { if (tail.stopReason !== "aborted" && tail.stopReason !== "error") return undefined; if (tail.content.some(block => block.type === "toolCall")) return undefined; end -= 1; } let start = end; while (start > 0 && messages[start - 1].role === "toolResult") start--; if (start === end) return undefined; const anchor = messages[start - 1]; if (anchor?.role !== "assistant") return undefined; if (anchor.stopReason !== "toolUse" && anchor.stopReason !== "stop") return undefined; const callIds = new Set(); for (const block of anchor.content) { if (block.type === "toolCall") callIds.add(block.id); } if (end - start !== callIds.size) return undefined; for (let i = start; i < end; i++) { const result = messages[i]; if (result.role !== "toolResult" || result.isError !== true || !callIds.has(result.toolCallId)) { return undefined; } } return start; } /** Result shape shared with automatic maintenance recovery. */ export interface RecoveryCompactionResult { continuationScheduled: boolean; automaticContinuationBlocked?: boolean; historyRewritten?: boolean; } type RequestBodyReadTimeoutRecovery = "not-applicable" | "handled-retry" | "handled-terminal"; /** Capabilities borrowed from the owning AgentSession. */ export interface TurnRecoveryHost { agent: Agent; sessionManager: SessionManager; settings: Settings; modelRegistry: ModelRegistry; configWarnings: string[]; model(): Model | undefined; /** * Whether the live context fits `model`'s usable window. `excludedMessage` * identifies a failed assistant turn that will be removed before retrying, so * selection judges the request that will actually be sent. See * `SessionMaintenance.contextFitsModel`. */ contextFitsModel(model: Model, excludedMessage?: AssistantMessage): boolean; /** Whether streamed text has already been committed to the active output sink. */ textOutputCommitted(): boolean; thinkingLevel(): ThinkingLevel | undefined; configuredThinkingLevel(): ConfiguredThinkingLevel | undefined; setThinkingLevel(level: ConfiguredThinkingLevel | undefined): void; /** Hard per-session effort ceiling; fallback recovery must never raise thinking above it. */ thinkingLevelCeiling(): Effort | undefined; /** The session never switches to a retry fallback model (a per-call `model` subagent). */ modelFallbackDisabled?(): boolean; isDisposed(): boolean; isStreaming(): boolean; isCompacting(): boolean; abortInProgress(): boolean; streamingEditAbortTriggered(): boolean; promptGeneration(): number; promptSequence(): number; /** * Live post-prompt abort signal; aborted when the user interrupts (Esc), * the turn is superseded, or the session is torn down. The unexpected-stop * judge wait links its timeout to this signal so a slow verdict never * blocks the abort drain. Optional so partial test stubs that never reach * the judge path need not provide it. */ unexpectedStopAbortSignal?(): AbortSignal; sessionId(): string; emitSessionEvent(event: AgentSessionEvent): Promise; scheduleAgentContinue(options: { source: string; delayMs?: number; generation?: number; shouldContinue?: () => boolean; onError?: (error: unknown) => void; }): void; waitForSessionMessagePersistence(message: AssistantMessage): Promise; appendSessionMessage(message: AssistantMessage): void; persistedAssistantEntryId(message: AssistantMessage): string | undefined; sessionMessageAlreadyPersisted(message: AssistantMessage): boolean; setModelWithProviderSessionReset(model: Model): Promise; /** Edit mode resolved for the active model and settings, captured before a fallback swap. */ resolveActiveEditMode(): EditModeSetting; /** * Rebuilds the model-dependent base system prompt after a swap. Retry fallback passes * `retryFallback`, which rebuilds only for an edit-mode change so the cached prompt survives. */ syncAfterModelChange(previousEditMode: EditModeSetting, options?: { retryFallback?: boolean }): Promise; resetCurrentResponsesProviderSession(reason: string): void; /** * Spend an eligible saved reset for the blocked provider pool. * `activeBlockUnblockAtMs` is the absolute unblock time parsed from the live * usage-limit error and never substitutes for live grant eligibility. */ maybeAutoRedeemReset(activeBlockUnblockAtMs?: number): Promise; runAutoCompaction( reason: "overflow" | "threshold" | "idle" | "incomplete", willRetry: boolean, options?: { autoContinue?: boolean; triggerContextTokens?: number; suppressContinuation?: boolean; suppressHandoff?: boolean; phase?: CodexCompactionContext["phase"]; terminalTextAnswer?: boolean; /** Skip `snapcompact` — a byte/payload-limit 413 recovery must not * retry with an even larger, media-heavy request (#11482). */ excludeMediaMethods?: boolean; }, ): Promise; shakeForRequestBodyReadTimeout(generation: number): Promise; withBashBranchTransition(operation: () => T): T; } /** Construction-time retry state restored from model selection. */ export interface TurnRecoveryOptions { initialRetryFallback?: InitialRetryFallbackState; /** Skip construction-time fallback-chain validation; the owner runs {@link TurnRecovery.validateRetryFallbackChains}. */ deferFallbackChainValidation?: boolean; /** * Override for the unexpected-stop judge verdict budget (default 15s). * Test seam so the slow-verdict boundary runs in milliseconds instead of * seconds; production never sets it. */ unexpectedStopJudgeTimeoutMs?: number; } type PendingRetryError = { entryId: string; persistenceKey: string; recovery: AssistantRetryRecoveryKind; attempt: number; note: string; }; /** How a retry saga's pending error entries are annotated when it closes. */ type RetryErrorCompletion = { status: "recovered"; supersedingMessage: AssistantMessage } | { status: "superseded" }; /** The `auto_retry_end` a closing retry saga publishes; its `retryErrors` come from the close itself. */ type RetrySagaEnd = Omit, "type" | "retryErrors">; /** How {@link TurnRecovery.handleRetryableError} may recover a failed turn. */ type RetryPolicyOptions = { allowModelFallback?: boolean; fireworksFastFallback?: boolean; hardErrorFallback?: boolean; preserveFailedTurn?: boolean; }; type UsageLimitOutcome = { switchedCredential: boolean; retryAfterMs: number; retryAtMs: number | undefined; blockedUntilMs: number | undefined; priorBlockedUntilMs: number | undefined; priorBlockedUntilTimed: boolean | undefined; reportResetAtMs: number | undefined; }; /** Request-time provider for the main Agent; not serialized into session history. */ export const kUsageLimitRequestProvider = Symbol("usage-limit-request-provider"); type IssuedProviderAssistantMessage = AssistantMessage & { [kUsageLimitRequestProvider]?: string }; /** Owns terminal-stop recovery, automatic retries, and fallback routing. */ export class TurnRecovery { readonly #host: TurnRecoveryHost; readonly #unexpectedStopJudgeTimeoutMs: number; #retryAbortController: AbortController | undefined; #retryAttempt = 0; #genericAbortStreak = 0; /** * Selectors a classifier refusal has already moved off in the current retry saga; see the walk * bound below. Cleared in `resolveRetry()`, which every end of a saga goes through (an answer, * the exhausted budget, a cancelled wait, an abort), so no turn inherits another turn's walk. */ readonly #refusalWalkTried = new Set(); /** Per-model account walk allowance, snapshotted once and cleared when the retry saga ends. */ readonly #accountPolicyRotationsRemaining = new Map(); #requestBodyReadTimeoutRecoveryPromptSequence: number | undefined; #retryPromise: Promise | undefined; #retryResolve: (() => void) | undefined; #activeRetryFallback: ActiveRetryFallbackState | undefined; #activeFallbackCreditRedemption?: { targetSelector: string; handle: AnthropicFallbackCreditHandle; }; #usageReserveApproval: { model: string; sessionId: string } | undefined; #pendingRetryErrors: PendingRetryError[] = []; #usageLimitOutcomes = new WeakMap>(); #emptyStopRetryCount = 0; #unexpectedStopRetryCount = 0; #malformedFunctionCallRetryCount = 0; #streamStallContinueCount = 0; #acceptTerminalEmptyStopForPrompt = false; // Three fields sit near the word "serve" and are deliberately distinct: // `#activeRetryFallback.served` gates the one-shot `retry_fallback_succeeded` // event for the current arm, `#fallbackRouted` says how the CURRENT model was // reached, and `#lastServed` is the session's attribution. A fallback flipping // to served does not by itself move attribution — only a settled turn does. /** * Attribution of the newest turn that produced output, tagged with the * session it belongs to. Anchoring rather than resetting follows * `#ensurePersistedMessageKeys`: every real switch mints a new session id, so * stale attribution drops itself and no mutation call site has to remember to * clear it. The id — not the file — is the anchor because an unpersisted * session has no file, and comparing two `undefined`s would never invalidate. */ #lastServed: { attribution: ServingModel; sessionId: string } | undefined; /** * Session whose current model was reached by fallback routing rather than by * the configured primary, or `undefined` when it was not. Tracked separately * from {@link #activeRetryFallback} because the Fireworks Fast degrade swaps * models without arming a chain, and anchored like {@link #lastServed}: * switching transcripts in place must not describe a fresh session's model * with how the previous one was routed. */ #fallbackRoutedFor: string | undefined; /** Memoized bootstrap answer, for the window before anything has served. */ #bootstrapCache: | { model: Model; level: ThinkingLevel | undefined; routed: boolean; value: ServingModel } | undefined; /** * Fallback-chain warnings pushed to `configWarnings` by startup validation, * tracked so a post-discovery re-check can retract the ones discovery * resolved (#10048). */ #fallbackChainWarnings = new Set(); /** Whether startup validation deferred any warning pending in-flight discovery (#10048). */ #pendingDiscoveryDeferredValidation = false; /** Whether startup fallback-chain validation has run; later checks go through the post-discovery reconcile. */ #fallbackChainsValidated = false; constructor(host: TurnRecoveryHost, options: TurnRecoveryOptions = {}) { this.#host = host; this.#unexpectedStopJudgeTimeoutMs = options.unexpectedStopJudgeTimeoutMs ?? UNEXPECTED_STOP_TIMEOUT_MS; if (options.initialRetryFallback) { this.#activeRetryFallback = { ...options.initialRetryFallback, lastAppliedFallbackThinkingLevel: host.configuredThinkingLevel(), pinned: options.initialRetryFallback.pinned ?? false, }; this.#markFallbackRouted(); } if (!options.deferFallbackChainValidation) this.validateRetryFallbackChains(); } /** Current automatic retry attempt. */ get attempt(): number { return this.#retryAttempt; } /** Promise settled when the active retry saga finishes. */ get retryPromise(): Promise | undefined { return this.#retryPromise; } /** Whether the CURRENT session's model was reached by fallback routing. */ get #fallbackRouted(): boolean { return ( this.#fallbackRoutedFor !== undefined && this.#fallbackRoutedFor === this.#host.sessionManager.getSessionId() ); } #markFallbackRouted(): void { this.#fallbackRoutedFor = this.#host.sessionManager.getSessionId(); } /** * Model this session's produced work is attributed to. * * A model switch is a routing decision, not evidence the target can produce * anything: a candidate that errors on its first request produced none of the * turns already in this session. So attribution only ever names a model that * has settled a turn here, and a switch — into a fallback, back to a restored * primary, or anywhere else — moves it only once the new model answers. * * Before anything has served there is no earlier work to miscredit, so the * configured model is both the only available answer and a safe one. */ get servingModel(): ServingModel | undefined { const served = this.#lastServed; if (served && served.sessionId === this.#host.sessionManager.getSessionId()) return served.attribution; const model = this.#host.model(); if (!model) return undefined; // Polled per streaming event and per render, so the pre-first-turn window // must not format a selector on every call. const level = this.#host.thinkingLevel(); const cached = this.#bootstrapCache; if (cached && cached.model === model && cached.level === level && cached.routed === this.#fallbackRouted) { return cached.value; } const value: ServingModel = { selector: formatRetryFallbackSelector(model, level), modelIdentity: formatModelStringWithRouting(model), thinkingLevel: level, isFallback: this.#fallbackRouted, contextWindow: model.contextWindow, }; this.#bootstrapCache = { model, level, routed: this.#fallbackRouted, value }; return value; } /** * Carries attribution onto a new session id that continues this conversation. * * The session-id anchor assumes a new id means an unrelated transcript, which * holds for `/new` and for resuming something else. A fork breaks that * assumption on purpose: it clones the transcript and keeps running the same * recovery state under a fresh id. Dropping attribution there would bootstrap * an unproven fallback as the primary and re-credit it with the work the * previous model did — the very bug the anchor exists to prevent. * * Only state belonging to `previousSessionId` moves, so an id left behind by * an earlier switch stays expired. */ reanchorServedAttribution(previousSessionId: string): void { const sessionId = this.#host.sessionManager.getSessionId(); if (this.#lastServed?.sessionId === previousSessionId) { this.#lastServed = { ...this.#lastServed, sessionId }; } if (this.#fallbackRoutedFor === previousSessionId) { this.#fallbackRoutedFor = sessionId; } } /** Resets per-prompt recovery counters and terminal-stop acceptance. */ resetForNewPrompt(): void { this.#emptyStopRetryCount = 0; this.#unexpectedStopRetryCount = 0; this.#malformedFunctionCallRetryCount = 0; this.#streamStallContinueCount = 0; this.#acceptTerminalEmptyStopForPrompt = false; this.#activeFallbackCreditRedemption = undefined; } /** Sets whether one terminal empty stop is accepted for the current prompt. */ setAcceptTerminalEmptyStop(accept: boolean): void { this.#acceptTerminalEmptyStopForPrompt = accept; } /** Consumes any queued Anthropic fallback credit handle for the retry turn. */ consumeActiveFallbackCreditRedemption(targetModel?: Model): AnthropicFallbackCreditHandle | undefined { const active = this.#activeFallbackCreditRedemption; if (!active) return undefined; if (targetModel && `${targetModel.provider}/${targetModel.id}` !== active.targetSelector) { return undefined; } this.#activeFallbackCreditRedemption = undefined; if (Date.now() >= active.handle.expiresAt) { return undefined; } return active.handle; } /** * Records which model produced this turn, re-arms the one-shot Responses * request-body-timeout recovery, marks an active fallback as having served, * then closes a successful retry saga and annotates recovered persisted * errors. */ async onAssistantSettledSuccessfully(message: AssistantMessage): Promise { if (!assistantTurnProducedOutput(message)) { return; } // A turn that produced output is forward progress: later history growth is // new, so the next exact full-replay body-read timeout gets its own single // shake-and-retry. Back-to-back timeouts never reach this point, so the // one-shot bound on an unchanged-request loop is unaffected. this.#requestBodyReadTimeoutRecoveryPromptSequence = undefined; const model = this.#host.model(); if (model) { const level = this.#host.thinkingLevel(); this.#lastServed = { attribution: { selector: formatRetryFallbackSelector(model, level), modelIdentity: formatModelStringWithRouting(model), thinkingLevel: level, isFallback: this.#fallbackRouted, contextWindow: model.contextWindow, }, sessionId: this.#host.sessionManager.getSessionId(), }; } // Independent of the retry saga below: a usage-aware fallback is applied // before a request without ever incrementing `#retryAttempt`, and it still // owns every turn it serves. Gating this on the saga left such a fallback // permanently unproven, hiding it from observers for the whole session. if (this.#activeRetryFallback && !this.#activeRetryFallback.served && model) { this.#activeRetryFallback.served = true; await this.#host.emitSessionEvent({ type: "retry_fallback_succeeded", model: this.#lastServed?.attribution.selector ?? formatRetryFallbackSelector(model, this.#host.thinkingLevel()), role: this.#activeRetryFallback.role, }); } if (this.#retryAttempt === 0) { return; } await this.#closeRetrySaga( { success: true, attempt: this.#retryAttempt }, { retryErrors: { status: "recovered", supersedingMessage: message } }, ); } /** Closes a failed retry saga when no compaction continuation took ownership. */ async onErrorSettledWithoutRetry(message: AssistantMessage, compaction: RecoveryCompactionResult): Promise { if (message.stopReason !== "error" || this.#retryAttempt === 0 || compaction.continuationScheduled) return; const attempt = this.#retryAttempt; this.#retryAttempt = 0; await this.#host.emitSessionEvent({ type: "auto_retry_end", success: false, attempt, finalError: message.errorMessage, }); this.#clearPendingRetryErrors(); } /** Persists an otherwise skipped terminal empty error turn. */ persistTerminalEmptyErrorTurn(message: AssistantMessage): Promise { return this.#persistTerminalEmptyErrorTurn(message); } /** Handles empty terminal assistant turns and schedules bounded recovery. */ handleEmptyAssistantStop(message: AssistantMessage): Promise<"continue" | "terminal" | undefined> { return this.#handleEmptyAssistantStop(message); } /** Classifies suspicious terminal stops and schedules bounded recovery. */ handleUnexpectedAssistantStop(message: AssistantMessage): Promise { return this.#handleUnexpectedAssistantStop(message); } /** * Continue past a provider-rejected function call that the replay-based * retry declined. Gemini reports `MALFORMED_FUNCTION_CALL` when the model * transcribes the call as text (`call:default_api:read{…}`, a tool_code * fence) instead of emitting a structured call; the text is already * rendered, so {@link isRetryableError} refuses to replay the turn and the * session stopped on a pinned error. Nothing ran and nothing needs * replaying: keep the failed turn in context so the model sees its own * output, append a corrective developer message, and resume. Bounded per * prompt; past the cap the error surfaces as before. */ handleMalformedFunctionCallStop(message: AssistantMessage): boolean { if (message.stopReason !== "error") return false; const id = this.#classifyRetryMessage(message); if (!AIError.is(id, AIError.Flag.MalformedFunctionCall)) { this.#malformedFunctionCallRetryCount = 0; return false; } if (this.#host.abortInProgress() || this.#host.isDisposed()) return false; this.#malformedFunctionCallRetryCount++; if (this.#malformedFunctionCallRetryCount > MALFORMED_FUNCTION_CALL_MAX_RETRIES) { logger.warn("Assistant kept emitting malformed function calls after retry cap", { attempts: this.#malformedFunctionCallRetryCount - 1, model: message.model, provider: message.provider, }); this.#malformedFunctionCallRetryCount = 0; return false; } logger.info("Malformed function call; continuing with corrective reminder", { attempt: this.#malformedFunctionCallRetryCount, model: message.model, provider: message.provider, }); this.#host.agent.appendMessage({ role: "developer", content: [ { type: "text", text: prompt.render(malformedFunctionCallRetryTemplate, { retryCount: this.#malformedFunctionCallRetryCount, maxRetries: MALFORMED_FUNCTION_CALL_MAX_RETRIES, }), }, ], attribution: "agent", timestamp: Date.now(), }); this.#host.scheduleAgentContinue({ source: "malformed-function-call-retry", generation: this.#host.promptGeneration(), }); return true; } /** * Continue past a mid-stream transport failure (idle stall, HTTP/2 reset, * premature close, socket closed mid-body) that hit a text-only turn after * its text rendered. {@link isRetryableError} refuses to replay committed * text, and {@link classifyResolvedInterruptedToolTurn} only resumes turns * with tool calls, so the session used to stop on a pinned error. Resuming * from the trailing assistant message would be a prefill, which newer Claude * models reject: keep the partial turn in context, append a developer * reminder to pick up where the text stopped, and continue. Honors * `retry.enabled`; bounded per prompt, past the cap the error surfaces as before. * A socket close counts here but not for tool turns: without tool calls there * is no executed side effect whose resumption policy it could change. So does a * Codex native-lane steer rejection: the server dropped the response over our * own `response.steer`, and the rejected steer drains as ordinary input next. */ handleCommittedTextStreamStall(message: AssistantMessage): boolean { const id = this.#classifyRetryMessage(message); const socketClosed = message.stopReason === "error" && AIError.retriable(id) && !this.#host.streamingEditAbortTriggered() && isUnexpectedSocketCloseMessage(message.errorMessage ?? ""); if (!socketClosed && !AIError.isCodexSteerRejection(message) && !this.#isMidStreamTransportFailure(message, id)) { this.#streamStallContinueCount = 0; return false; } if (!this.autoRetryEnabled || this.#host.abortInProgress() || this.#host.isDisposed()) return false; if (!this.#host.textOutputCommitted()) return false; let hasText = false; for (const block of message.content) { if (block.type === "toolCall" || block.type === "image" || block.type === "anthropicServerTool") return false; if (block.type === "text" && hasNonWhitespace(block.text)) hasText = true; } if (!hasText) return false; this.#streamStallContinueCount++; if (this.#streamStallContinueCount > STREAM_STALL_CONTINUE_MAX_RETRIES) { logger.warn("Stream kept stalling after committed text past retry cap", { attempts: this.#streamStallContinueCount - 1, model: message.model, provider: message.provider, }); this.#streamStallContinueCount = 0; return false; } logger.info("Stream failed after committed text; continuing with resume reminder", { attempt: this.#streamStallContinueCount, model: message.model, provider: message.provider, errorMessage: message.errorMessage, }); this.#host.agent.appendMessage({ role: "developer", content: [ { type: "text", text: prompt.render(streamStallContinueTemplate, { retryCount: this.#streamStallContinueCount, maxRetries: STREAM_STALL_CONTINUE_MAX_RETRIES, }), }, ], attribution: "agent", timestamp: Date.now(), }); this.#host.scheduleAgentContinue({ source: "stream-stall-continue", generation: this.#host.promptGeneration(), }); return true; } /** Removes a persisted failed assistant turn after its persistence slot settles; returns the dropped branch entry id. */ dropPersistedAssistantTurn(message: AssistantMessage): Promise { return this.#dropPersistedAssistantTurn(message); } /** Runs recovery compaction and restores the failed turn when no rewrite occurs. */ runRecoveryCompactionWithRollback( reason: "overflow" | "incomplete", message: AssistantMessage, options: { autoContinue: boolean; triggerContextTokens?: number; excludeMediaMethods?: boolean }, ): Promise { return this.#runRecoveryCompactionWithRollback(reason, message, options); } /** * Restores the configured primary after fallback cooldown expiry. * @returns true when the active model was actually switched back to the * primary, so callers can re-run the pre-send context-fit check against the * reverted (possibly smaller) window before issuing the next request. */ maybeRestoreRetryFallbackPrimary(): Promise { return this.#maybeRestoreRetryFallbackPrimary(); } /** Applies model fallback policy from live usage health before a turn starts. */ maybeApplyUsageAwareFallback(signal: AbortSignal, confirmer?: UsageFallbackConfirmer): Promise { return this.#maybeApplyUsageAwareFallback(signal, confirmer); } /** Applies automatic retry, credential rotation, and model fallback policy. */ handleRetryableError(message: AssistantMessage, options?: RetryPolicyOptions): Promise { return this.#handleRetryableError(message, options); } /** * Records a usage-limit failure before replay eligibility decides whether the * failed turn may be discarded. Returns whether credential recovery switched * the active account. */ async recordUsageLimitOutcome(message: AssistantMessage): Promise { if (message.stopReason !== "error") return false; const id = this.#classifyRetryMessage(message); const activeModel = this.#host.model(); if (!activeModel || !AIError.is(id, AIError.Flag.UsageLimit)) return false; // Only the request-time provider proves a delayed error belongs to the old pool. const requestProvider = (message as IssuedProviderAssistantMessage)[kUsageLimitRequestProvider]; if (requestProvider !== undefined && requestProvider !== activeModel.provider) return false; let recorded = this.#usageLimitOutcomes.get(message); if (!recorded) { const errorMessage = message.errorMessage || "Unknown error"; const parsedRetryAfterMs = this.#parseRetryAfterMsFromError(errorMessage); const retryAfterMs = parsedRetryAfterMs ?? calculateRateLimitBackoffMs(parseRateLimitReason(errorMessage)); recorded = (async (): Promise => { const outcome = await this.#host.modelRegistry.authStorage.limits.markReached( activeModel.provider, this.#host.sessionId(), { retryAfterMs, // Provider-stated timing only when the error text parsed; // the 30-minute fallback is a guess. providerTimed: parsedRetryAfterMs !== undefined, baseUrl: activeModel.baseUrl, modelId: activeModel.id, }, ); return { switchedCredential: outcome.switched, retryAfterMs, retryAtMs: outcome.retryAtMs, blockedUntilMs: outcome.blockedUntilMs, priorBlockedUntilMs: outcome.priorBlockedUntilMs, priorBlockedUntilTimed: outcome.priorBlockedUntilTimed, reportResetAtMs: outcome.reportResetAtMs, }; })(); this.#usageLimitOutcomes.set(message, recorded); } return (await recorded).switchedCredential; } /** Prompts after transient overlap with a prior agent run. `onClaimed` fires synchronously * right after a successful `agent.prompt` claim (before the run is awaited), letting the * caller mark the exact moment its own run went live. */ promptAgentWithIdleRetry( messages: AgentMessage[], options?: { toolChoice?: ToolChoice }, onClaimed?: () => void, ): Promise { return this.#promptAgentWithIdleRetry(messages, options, onClaimed); } /** Parses provider retry and rate-limit reset hints into a delay. */ parseRetryAfterMsFromError(errorMessage: string): number | undefined { return this.#parseRetryAfterMsFromError(errorMessage); } /** * The model a refusal walk records: `provider/id` without the thinking level, so the current * selector (`…:high`) and the bare chain entries naming the same model match. */ #refusalWalkKey(selector: string): string { const parsed = parseRetryFallbackSelector(selector, this.#host.modelRegistry); return parsed ? formatRetryFallbackBaseSelector(parsed) : selector; } /** Resolve the pending retry promise; the saga is over, and so is its refusal walk. */ resolveRetry(): void { this.#refusalWalkTried.clear(); this.#accountPolicyRotationsRemaining.clear(); if (this.#retryResolve) { this.#retryResolve(); this.#retryResolve = undefined; this.#retryPromise = undefined; } } /** * Ends a retry saga: records its outcome in the session (the terminal empty * error turn, the annotations on the attempts it retried), publishes * `auto_retry_end`, then resets the saga and resolves `retryPromise`. * Publishing and releasing run in `finally`, so a session write that throws * (a full disk, an unwritable transcript) still ends the saga before its * error propagates. Left pending, `retryPromise` keeps the in-flight prompt * waiting for a retry that never runs: the session reports streaming for * good and queues every later message as a steer that never drains. */ async #closeRetrySaga( end: RetrySagaEnd, record: { terminalTurn?: AssistantMessage; retryErrors?: RetryErrorCompletion }, ): Promise { let retryErrors: RetryErrorUpdate[] | undefined; try { if (record.terminalTurn) await this.persistTerminalEmptyErrorTurn(record.terminalTurn); if (record.retryErrors) retryErrors = await this.#markPendingRetryErrors(record.retryErrors); } finally { try { await this.#host.emitSessionEvent({ type: "auto_retry_end", ...end, ...(retryErrors ? { retryErrors } : {}), }); } finally { this.#clearPendingRetryErrors(); this.#retryAttempt = 0; this.resolveRetry(); } } } #clearPendingRetryErrors(): void { this.#pendingRetryErrors = []; } /** * Durably record a terminal empty error turn (`stopReason: "error"` with no * substantive content) that `#persistSessionMessageIfMissing` skipped, so the * session JSONL keeps a record of why the run stopped instead of ending at the * last tool result. A no-op for non-empty/non-error turns and idempotent via * the already-persisted guard; the turn is dropped from active context by the * caller (or `isProviderRefusalMessage`/`isEmptyErrorTurn` filters) so it is * never replayed on the wire. Used by the retry-lifecycle dead-ends and the * non-retry terminal error tail. */ async #persistTerminalEmptyErrorTurn(message: AssistantMessage): Promise { await this.#host.waitForSessionMessagePersistence(message); if (!isEmptyErrorTurn(message)) return; if (this.#host.sessionMessageAlreadyPersisted(message)) return; this.#host.appendSessionMessage(message); } #retryRecoveryKind( id: number, switchedCredential: boolean, switchedModel: boolean, delayMs: number, ): AssistantRetryRecoveryKind { if (switchedCredential) return "credential"; if (switchedModel) return "model"; if (AIError.is(id, AIError.Flag.UsageLimit) && delayMs > 0) return "wait"; return "plain"; } #retryRecoveryNote(recovery: AssistantRetryRecoveryKind, rateLimited: boolean): string { const parts: string[] = []; if (rateLimited) { parts.push("rate-limited"); } else if (recovery === "plain") { parts.push("error"); } if (recovery === "credential") { parts.push("switched account"); } else if (recovery === "model") { parts.push("switched model"); } else if (recovery === "wait") { parts.push("waited"); } parts.push("retried"); return parts.join("; "); } async #recordPendingRetryError( message: AssistantMessage, id: number, options: { switchedCredential: boolean; switchedModel: boolean; delayMs: number }, ): Promise { await this.persistTerminalEmptyErrorTurn(message); const persistenceKey = sessionMessagePersistenceKey(message); if (!persistenceKey) return; const branch = this.#host.sessionManager.getBranchView(); const branchEntry = branch.findLast( entry => entry.type === "message" && entry.message.role === "assistant" && sessionMessagePersistenceKey(entry.message) === persistenceKey && (sameMessageContent(entry.message, message) || this.#isSameAssistantMessage(entry.message, message)), ); if (!branchEntry) return; if (this.#pendingRetryErrors.some(error => error.entryId === branchEntry.id)) return; const rateLimited = AIError.is(id, AIError.Flag.UsageLimit); const recovery = this.#retryRecoveryKind(id, options.switchedCredential, options.switchedModel, options.delayMs); const note = this.#retryRecoveryNote(recovery, rateLimited); this.#pendingRetryErrors.push({ entryId: branchEntry.id, persistenceKey, recovery, attempt: this.#retryAttempt, note, }); } async #markPendingRetryErrors(completion: RetryErrorCompletion): Promise { if (this.#pendingRetryErrors.length === 0) return []; const branch = this.#host.sessionManager.getBranchView(); const branchById = new Map(); for (const entry of branch) { branchById.set(entry.id, entry); } const retryErrors: RetryErrorUpdate[] = []; for (const pending of this.#pendingRetryErrors) { let entry = branchById.get(pending.entryId); if (entry?.type !== "message" || entry.message.role !== "assistant") { entry = branch.findLast( candidate => candidate.type === "message" && candidate.message.role === "assistant" && sessionMessagePersistenceKey(candidate.message) === pending.persistenceKey, ); } if (entry?.type !== "message" || entry.message.role !== "assistant") continue; let retryRecovery: AssistantRetryRecovery; if (completion.status === "recovered") { retryRecovery = { kind: "auto-retry", status: "recovered", attempt: pending.attempt, recoveredAt: new Date().toISOString(), recovery: pending.recovery, note: pending.note, supersededBy: { timestamp: completion.supersedingMessage.timestamp, ...(completion.supersedingMessage.responseId === undefined ? {} : { responseId: completion.supersedingMessage.responseId }), provider: completion.supersedingMessage.provider, model: completion.supersedingMessage.model, }, }; } else { retryRecovery = { kind: "auto-retry", status: "superseded", attempt: pending.attempt, recovery: pending.recovery, note: pending.note, }; } entry.message.retryRecovery = retryRecovery; retryErrors.push({ entryId: entry.id, persistenceKey: pending.persistenceKey, note: retryRecovery.note, retryRecovery, }); } if (retryErrors.length > 0) { await this.#host.sessionManager.rewriteEntries(); } return retryErrors; } #isRecoverableProviderEmptyOutput(message: AssistantMessage): boolean { if (message.stopReason !== "error") return false; const id = this.#classifyRetryMessage(message); if (!AIError.is(id, AIError.Flag.EmptyResponse)) return false; return message.content.every( block => block.type === "thinking" || (block.type === "text" && !hasNonWhitespace(block.text)), ); } async #handleEmptyAssistantStop(assistantMessage: AssistantMessage): Promise<"continue" | "terminal" | undefined> { const providerEmptyOutput = this.#isRecoverableProviderEmptyOutput(assistantMessage); if (!isEmptyAssistantStop(assistantMessage) && !providerEmptyOutput) { this.#emptyStopRetryCount = 0; return undefined; } if (this.#acceptTerminalEmptyStopForPrompt && assistantMessage.stopReason === "stop") { this.#acceptTerminalEmptyStopForPrompt = false; this.#discardAcceptedTerminalEmptyStop(assistantMessage); this.#emptyStopRetryCount = 0; return undefined; } this.#emptyStopRetryCount++; if (this.#emptyStopRetryCount > EMPTY_STOP_MAX_RETRIES) { const attempts = this.#emptyStopRetryCount - 1; const outputTokens = assistantMessage.usage.output; const outputTokensExcludingKnownReasoning = Math.max( 0, outputTokens - (assistantMessage.usage.reasoningTokens ?? 0), ); let finalError: string; if (providerEmptyOutput) { finalError = "Assistant returned no final output after retry cap; try switching models"; } else if (outputTokensExcludingKnownReasoning > 0 && assistantMessage.content.length === 0) { // Report the usage without inferring why no deliverable content arrived. const reasoningTokens = assistantMessage.usage.reasoningTokens; const reasoningDetail = reasoningTokens === undefined ? "" : ` (${reasoningTokens} reasoning token${reasoningTokens === 1 ? "" : "s"})`; finalError = `Assistant returned an empty stop after retry cap; provider reported ${outputTokens} output token${outputTokens === 1 ? "" : "s"}${reasoningDetail}, but no text or tool calls were delivered. Usage alone does not establish the cause`; } else { finalError = "Assistant returned empty stop after retry cap; try switching models or `/shake images` to remove archived frames"; } assistantMessage.errorMessage = finalError; if (providerEmptyOutput) assistantMessage.errorId = AIError.create(); logger.warn(finalError, { attempts, model: assistantMessage.model, provider: assistantMessage.provider, outputTokens, }); await this.#host.emitSessionEvent({ type: "auto_retry_end", success: false, attempt: this.#retryAttempt > 0 ? this.#retryAttempt : attempts, finalError, }); this.#clearPendingRetryErrors(); this.#retryAttempt = 0; this.resolveRetry(); // A turn with no actionable output carries no transcript value, while its // provider usage can anchor the next prompt at the full failed-request size // and re-trigger compaction at the same boundary. Remove every capped // empty output; toolUse orphans still need this for Anthropic history. await this.#dropAssistantTurnDurably(assistantMessage); return "terminal"; } // The reparented leaf must be durably persisted before the retry continues: // the loader rebuilds the active branch from the last physical entry, so an // in-memory-only reparent lets the empty stop resurface on reload or after // a mid-retry process kill. await this.#dropAssistantTurnDurably(assistantMessage); this.#host.agent.appendMessage({ role: "developer", content: [{ type: "text", text: this.#emptyStopRetryReminder() }], attribution: "agent", timestamp: Date.now(), }); this.#host.scheduleAgentContinue({ source: "empty-stop-retry", generation: this.#host.promptGeneration(), }); return "continue"; } #emptyStopRetryReminder(): string { return prompt.render(emptyStopRetryTemplate, { retryCount: this.#emptyStopRetryCount, maxRetries: EMPTY_STOP_MAX_RETRIES, }); } async #handleUnexpectedAssistantStop(assistantMessage: AssistantMessage): Promise { const mode = cfgFeaturesUnexpectedStopDetection.get(this.#host.settings); if (mode === "none") { return false; } if (!isUnexpectedStopCandidate(assistantMessage)) { this.#unexpectedStopRetryCount = 0; return false; } const text = assistantMessage.content .filter((content): content is TextContent => content.type === "text") .map(content => content.text) .join("\n"); // A thinking-only terminal turn has no visible assistant message, so both // mechanical and smart modes retry it directly, whether or not the provider // exposed its reasoning text. Only a turn with visible text is gated on the // mode and the judge. Tool-call turns never reach this path: // isUnexpectedStopCandidate excludes them, including forced tools. if (hasNonWhitespace(text)) { if (mode === "mechanical") { this.#unexpectedStopRetryCount = 0; return false; } // This turn answers a stop nudge, which demands the next tool call or // the final answer. Visible text without a tool call is that final // answer; judging it again only chains no-op "nothing more to add" turns. if (this.#unexpectedStopRetryCount > 0) { this.#unexpectedStopRetryCount = 0; return false; } const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), this.#unexpectedStopJudgeTimeoutMs); // Esc/session teardown must interrupt the extended judge wait: link the // live session abort so abort() drains agent_end maintenance instead of // blocking on a verdict the user no longer wants. const sessionSignal = this.#host.unexpectedStopAbortSignal?.(); const onSessionAbort = (): void => controller.abort(); sessionSignal?.addEventListener("abort", onSessionAbort, { once: true }); const stop = { message: text, request: this.#latestOperatorRequest() }; let classification: boolean | undefined; try { classification = await classifyUnexpectedStop(stop, { settings: this.#host.settings, registry: this.#host.modelRegistry, sessionId: this.#host.sessionId(), model: this.#host.model() ?? undefined, metadataResolver: (provider: string) => this.#host.agent.metadataForProvider(provider), onUsage: journalJudgmentUsage(this.#host.sessionManager), telemetry: this.#host.agent.telemetry, signal: controller.signal, }); } finally { clearTimeout(timeout); sessionSignal?.removeEventListener("abort", onSessionAbort); } // The classifier maps aborts to undefined (no-retry); a session abort // during the wait must also skip the retry counter and the nudge. if (classification !== true || sessionSignal?.aborted === true) { this.#unexpectedStopRetryCount = 0; return false; } } this.#unexpectedStopRetryCount++; if (this.#unexpectedStopRetryCount > UNEXPECTED_STOP_MAX_RETRIES) { logger.warn("Assistant returned unexpected stop after retry cap", { attempts: this.#unexpectedStopRetryCount - 1, model: assistantMessage.model, provider: assistantMessage.provider, }); this.#unexpectedStopRetryCount = 0; return false; } this.#host.agent.appendMessage({ role: "developer", content: [{ type: "text", text: this.#unexpectedStopRetryReminder() }], attribution: "agent", timestamp: Date.now(), }); this.#host.scheduleAgentContinue({ source: "unexpected-stop-retry", generation: this.#host.promptGeneration(), }); return true; } /** The operator's latest typed request, which a text-only stop is judged against. */ #latestOperatorRequest(): string | undefined { const messages = this.#host.agent.state.messages; for (let i = messages.length - 1; i >= 0; i--) { const request = operatorTitleText(messages[i]!); if (request !== undefined) return request; } return undefined; } #unexpectedStopRetryReminder(): string { return prompt.render(unexpectedStopRetryTemplate, { retryCount: this.#unexpectedStopRetryCount, maxRetries: UNEXPECTED_STOP_MAX_RETRIES, }); } removeAssistantMessageFromActiveContext( assistantMessage: AssistantMessage, reason = "assistant-context-cleanup", ): void { const messages = this.#host.agent.state.messages; const lastMessage = messages[messages.length - 1]; const lastAssistant: AssistantMessage | undefined = lastMessage?.role === "assistant" ? lastMessage : undefined; if (lastAssistant !== undefined && this.#isSameAssistantMessage(lastAssistant, assistantMessage)) { this.#host.agent.replaceMessages(messages.slice(0, -1)); return; } // A miss means the failed turn is still in active context (or was never // there); log just enough to explain why the identity check failed. logger.debug("agent active context assistant removal missed", { reason, lastRole: lastMessage?.role, candidateTimestamp: assistantMessage.timestamp, lastTimestamp: lastAssistant?.timestamp, candidateStopReason: assistantMessage.stopReason, lastStopReason: lastAssistant?.stopReason, }); } /** * Drop a recoverable assistant turn from the persisted session branch once a * recovery path (context promotion or compaction) is committed. Waits for the * in-flight `message_end` persistence slot first so the branch entry exists * before we reparent past it. Active context removal is the caller's * responsibility — recovery paths clear it eagerly so the retry never * replays the failed turn, while no-recovery paths leave the persisted entry * (and the user-visible transcript line) in place. */ async #dropPersistedAssistantTurn(assistantMessage: AssistantMessage): Promise { await this.#host.waitForSessionMessagePersistence(assistantMessage); return this.discardAssistantTurn(assistantMessage); } /** * Drop the failed turn from active context and the persisted branch, then * durably persist the reparented leaf so the empty stop cannot resurface on * reload or after a mid-retry process kill. */ async #dropAssistantTurnDurably(assistantMessage: AssistantMessage): Promise { const droppedEntryId = await this.#dropPersistedAssistantTurn(assistantMessage); if (droppedEntryId) await this.#host.sessionManager.discardEntryDurably(droppedEntryId); } /** * Drop the failed assistant turn from persisted history, run * {@link #runAutoCompaction} for an `overflow` / `incomplete` recovery, and * restore the assistant entry if compaction did not actually commit * anything (no usable model/preparation, hook cancel, compaction error, * or a no-progress automatic-continuation block before any summary was * written). * * Compaction has to see a clean branch — otherwise its `prepareCompaction` * pass would keep the failed turn in the kept region and the retry would * replay it. But a return that was not paired with a fresh compaction * summary or a successful history rewrite means no recovery is in progress, * even if queued user input gets drained next. Restoring the failed turn * before that continuation preserves the visible stop reason and rebuilds the * active assistant tail that `Agent.continue()` needs to dequeue follow-ups. */ async #runRecoveryCompactionWithRollback( reason: "overflow" | "incomplete", assistantMessage: AssistantMessage, options: { autoContinue: boolean; triggerContextTokens?: number; excludeMediaMethods?: boolean }, ): Promise { const compactionEntryBefore = getLatestCompactionEntry(this.#host.sessionManager.getBranch()); await this.dropPersistedAssistantTurn(assistantMessage); const result = await this.#host.runAutoCompaction(reason, true, { autoContinue: options.autoContinue, triggerContextTokens: options.triggerContextTokens, phase: "mid_turn", excludeMediaMethods: options.excludeMediaMethods, }); const compactionEntryAfter = getLatestCompactionEntry(this.#host.sessionManager.getBranch()); if (result.historyRewritten !== true && compactionEntryAfter === compactionEntryBefore) { this.#restoreFailedAssistantTurn(assistantMessage); } return result; } #restoreFailedAssistantTurn(assistantMessage: AssistantMessage): void { if (!isEmptyErrorTurn(assistantMessage)) this.#host.sessionManager.appendMessage(assistantMessage); const lastMessage = this.#host.agent.state.messages.at(-1); if ( lastMessage?.role === "assistant" && this.#isSameAssistantMessage(lastMessage as AssistantMessage, assistantMessage) ) { return; } this.#host.agent.appendMessage(assistantMessage); } #discardAcceptedTerminalEmptyStop(assistantMessage: AssistantMessage): void { const branch = this.#host.sessionManager.getBranchView(); const branchIndex = branch.findLastIndex( entry => entry.type === "message" && entry.message.role === "assistant" && this.#isSameAssistantMessage(entry.message, assistantMessage), ); const branchEntry = branchIndex >= 0 ? branch[branchIndex] : undefined; // A branch entry's parent is the entry before it on the branch. const parentEntry = branchEntry?.parentId === null || branchEntry?.parentId === undefined ? undefined : branch[branchIndex - 1]; const prunePrompt = parentEntry?.type === "custom_message"; this.removeAssistantMessageFromActiveContext(assistantMessage, "accepted-terminal-empty-stop"); if (prunePrompt && this.#host.agent.state.messages.at(-1)?.role === "custom") { this.#host.agent.replaceMessages(this.#host.agent.state.messages.slice(0, -1)); } if (!branchEntry) return; const targetParentId = prunePrompt ? parentEntry.parentId : branchEntry.parentId; this.#host.withBashBranchTransition(() => { if (targetParentId === null) { this.#host.sessionManager.resetLeaf(); } else { this.#host.sessionManager.branch(targetParentId); } }); this.#host.sessionManager.appendCustomEntry("accepted-terminal-empty-stop"); } /** * Drop an assistant turn from BOTH the live agent context and the persisted * session branch (reparenting the leaf to the turn's parent), so a discarded * turn does not resurface on reload. Used for empty/reasoning-only stops and * the Gemini header-runaway interrupt, which must not replay a partial, * loop-fueling thinking block. */ discardAssistantTurn(assistantMessage: AssistantMessage): string | undefined { this.removeAssistantMessageFromActiveContext(assistantMessage); const branch = this.#host.sessionManager.getBranchView(); const persistedEntryId = this.#host.persistedAssistantEntryId(assistantMessage); const branchEntry = (persistedEntryId === undefined ? undefined : branch.find( entry => entry.id === persistedEntryId && entry.type === "message" && entry.message.role === "assistant", )) ?? branch.findLast( entry => entry.type === "message" && entry.message.role === "assistant" && this.#isSameAssistantMessage(entry.message as AssistantMessage, assistantMessage), ); if (!branchEntry) { return undefined; } this.#host.withBashBranchTransition(() => { if (branchEntry.parentId === null) { this.#host.sessionManager.resetLeaf(); } else { this.#host.sessionManager.branch(branchEntry.parentId); } }); return branchEntry.id; } #isSameAssistantMessage(left: AssistantMessage, right: AssistantMessage): boolean { return ( left === right || (left.timestamp === right.timestamp && left.provider === right.provider && left.model === right.model && left.stopReason === right.stopReason && left.errorMessage === right.errorMessage && Bun.hash(JSON.stringify(left.content)) === Bun.hash(JSON.stringify(right.content))) ); } /** * Classify retry decisions against the active session model. Test stream * shims and provider adapters can emit generic assistant metadata, but retry * policy belongs to the model that was actually requested for this turn. */ #classifyRetryMessage(message: AssistantMessage): number { const activeModel = this.#host.model(); if (!activeModel || message.api === activeModel.api) { return AIError.classifyMessage(message); } const id = AIError.classifyMessage({ api: activeModel.api, errorId: message.errorId, errorMessage: message.errorMessage, errorStatus: message.errorStatus, }); message.errorId = id; return id; } #isUsagePreflightBlocked(message: AssistantMessage): boolean { return message.errorMessage?.startsWith(USAGE_PREFLIGHT_BLOCKED_PREFIX) === true; } /** * Retry an empty, reason-less provider abort: a turn with no content that * carries the generic sentinel (bare `abort()`), whether the provider * finalized it as `stopReason: "aborted"` or leaked it as `stopReason: * "error"` (a stalled/dropped stream reported as an error rather than an * abort — issue #5375). Only fires while the session is neither aborting nor * tearing down. A user/lifecycle abort (`#abortInProgress`), a dispose-driven * abort (`#isDisposed`), or a session-induced streaming-edit guard abort * (`StreamingEditGuard.abortTriggered` — failed-patch * preview) is deliberate and MUST settle the turn instead: routing it through * retry would orphan `#retryPromise` on a continuation the guard skips * (hanging the in-flight `prompt()`) or silently undo the guard's intended * abort. Deliberate user interrupts (`UserInterrupt`) and silent aborts carry * their own marker, not the generic sentinel, so they never match here. */ isRetryableReasonlessAbort(message: AssistantMessage): boolean { if ( (message.stopReason !== "aborted" && message.stopReason !== "error") || // A `fallback` boundary is bookkeeping, not output: a stream aborted // right after a mid-turn server-side fallback persists `[fallback]`, // which is as replay-safe to retry as an empty turn (`isEmptyErrorTurn` // already counts the marker as no output). message.content.some(block => block.type !== "fallback") || this.#host.abortInProgress() || this.#host.isDisposed() || this.#host.streamingEditAbortTriggered() ) { return false; } const id = this.#classifyRetryMessage(message); if (message.stopReason === "aborted" && AIError.is(id, AIError.Flag.Abort)) return true; if (message.errorMessage === undefined || !Object.hasOwn(GENERIC_ABORT_MESSAGES, message.errorMessage)) { return false; } message.errorId = AIError.create(AIError.Flag.Abort); return true; } /** * Own the exact Responses request-body-read timeout so terminal special outcomes * cannot fall through to the generic 408 replay path. */ async handleResponsesRequestBodyReadTimeout(message: AssistantMessage): Promise { if (message.stopReason !== "error" || !AIError.isResponsesRequestBodyReadTimeout(message)) { return "not-applicable"; } const generation = this.#host.promptGeneration(); const promptSequence = this.#host.promptSequence(); const replayUnsafe = this.#hasReplayUnsafeOutput(message); const terminal = (): RequestBodyReadTimeoutRecovery => { if (!replayUnsafe) this.removeAssistantMessageFromActiveContext(message, "request-body-timeout-terminal"); return "handled-terminal"; }; const retrySettings = cfgRetry.get(this.#host.settings); if ( this.#requestBodyReadTimeoutRecoveryPromptSequence === promptSequence || !retrySettings.enabled || retrySettings.maxRetries <= this.#retryAttempt || this.#host.isDisposed() || this.#host.abortInProgress() || this.#host.isCompacting() || replayUnsafe ) { return terminal(); } this.#requestBodyReadTimeoutRecoveryPromptSequence = promptSequence; const shook = await this.#host.shakeForRequestBodyReadTimeout(generation); if ( !shook || this.#host.promptGeneration() !== generation || this.#host.isDisposed() || this.#host.abortInProgress() ) { return terminal(); } return (await this.#handleRetryableError(message, { allowModelFallback: false })) ? "handled-retry" : terminal(); } /** * Check if an error is retryable (transient errors, usage limits, or * account-scoped policy denials that can rotate credentials). * Context overflow is NOT retryable (handled by compaction instead). */ isRetryableError(message: AssistantMessage): boolean { if (message.stopReason !== "error") return false; if (this.#isUsagePreflightBlocked(message)) return false; if (AIError.isResponsesRequestBodyReadTimeout(message)) return false; const model = this.#host.model(); const immutableAnthropicThinkingError = model?.api === "anthropic-messages" && (message.errorStatus === 400 || message.errorId === 400 || message.errorMessage?.startsWith("400 ") === true) && AIError.isImmutableLatestThinkingText(message.errorMessage ?? ""); if (immutableAnthropicThinkingError) return false; const id = this.#classifyRetryMessage(message); // Context overflow is handled by compaction, not retry. const contextWindow = this.#host.model()?.contextWindow ?? 0; if (AIError.isContextOverflow(message, contextWindow)) return false; // Credential rotation and classifier fallbacks are safe only before // committed text, images, tool calls, or server tools. Thinking-only // output remains replay-safe. A classifier refusal, malformed-function // response, or retriable transport error (the provider stream died // mid-turn after a complete tool call — e.g. a socket close — and every // emitted call was paired with a synthetic `executed: false` result) may // also be replayed when every emitted tool call is paired with positive // proof that it never executed. const replaySafeUnexecutedTools = (this.isClassifierRefusal(message) || AIError.is(id, AIError.Flag.MalformedFunctionCall) || AIError.retriable(id)) && this.#unexecutedToolCallsReplaySafe(message); if (this.#hasReplayUnsafeOutput(message) && !replaySafeUnexecutedTools) return false; if (AIError.is(id, AIError.Flag.AccountPolicy) || this.isClassifierRefusal(message)) return true; return AIError.retriable(id); } /** * True when every emitted tool call provably never executed and no other * replay-unsafe output exists. The caller restricts this exception to * classifier refusals, malformed-function responses, and retriable * transport errors. * * Gemini can report `MALFORMED_FUNCTION_CALL` after streaming an earlier, * well-formed call. Anthropic classifiers can likewise refuse after a call. * The agent loop pairs each emitted-but-unrun call with a synthetic * `executed: false` result, which proves `tool.execute()` never ran. * * Any uncertainty keeps the replay veto in place: the assistant must exist * in state, every call must have a later synthetic result, every result must * say `executed === false`, and the turn must contain no image, server tool, * or committed non-whitespace text. */ #unexecutedToolCallsReplaySafe(message: AssistantMessage): boolean { const emittedToolCallIds = new Set(); for (const block of message.content) { if (block.type === "toolCall") { emittedToolCallIds.add(block.id); continue; } if (block.type === "image" || block.type === "anthropicServerTool") return false; if (block.type === "text" && this.#host.textOutputCommitted() && hasNonWhitespace(block.text)) return false; } if (emittedToolCallIds.size === 0) return false; // The errored assistant message is NOT the tail of state: the agent loop // appends the synthetic results after it before the turn ends, so locate it // by walking backwards exactly as `classifyResolvedInterruptedToolTurn` does. const messages = this.#host.agent.state.messages; let assistantIndex = -1; for (let i = messages.length - 1; i >= 0; i--) { const candidate = messages[i]; if (candidate.role === "assistant" && this.#isSameAssistantMessage(candidate, message)) { assistantIndex = i; break; } } if (assistantIndex < 0) return false; const unexecutedToolCallIds = new Set(); for (let i = assistantIndex + 1; i < messages.length; i++) { const candidate = messages[i]; if (candidate.role !== "toolResult" || !emittedToolCallIds.has(candidate.toolCallId)) continue; // Every result for an emitted call is inspected, not just the first: a // real result anywhere in the tail means the tool ran. if (!isSyntheticToolResultMessage(candidate) || candidate.details?.executed !== false) return false; unexecutedToolCallIds.add(candidate.toolCallId); } return unexecutedToolCallIds.size === emittedToolCallIds.size; } /** * Classify a reasonless abort, idle stream stall, HTTP/2 stream reset, or * premature stream close whose emitted tool calls all have results. The failed * assistant/tool-result pair stays in context so continuation cannot replay * completed side effects; synthetic results tell the next turn that an * unexecuted call must be reissued. */ classifyResolvedInterruptedToolTurn(message: AssistantMessage): "reasonless-abort" | "stream-stall" | undefined { const id = this.#classifyRetryMessage(message); const genericAbort = message.errorMessage !== undefined && Object.hasOwn(GENERIC_ABORT_MESSAGES, message.errorMessage); const reasonlessAbort = (message.stopReason === "aborted" || message.stopReason === "error") && !this.#host.abortInProgress() && !this.#host.isDisposed() && !this.#host.streamingEditAbortTriggered() && ((message.stopReason === "aborted" && AIError.is(id, AIError.Flag.Abort)) || genericAbort); if (!reasonlessAbort && !this.#isMidStreamTransportFailure(message, id)) return undefined; if (reasonlessAbort && genericAbort) message.errorId = AIError.create(AIError.Flag.Abort); // Idle stall and HTTP/2 RST both close the Cursor Connect stream: // the lazy watchdog aborts the request signal, and cursor.ts then // calls `h2Request.close()`. There is no in-flight server exec to // race, so unmarked MCP/todo blocks can continue once every emitted // call has a matching result. A reasonless abort ends the turn and // the agent loop pairs leftover calls with `executed: false`. const resolvedToolCallIds: string[] = []; for (const block of message.content) { if (block.type !== "toolCall") continue; resolvedToolCallIds.push(block.id); } if (resolvedToolCallIds.length === 0) return undefined; const messages = this.#host.agent.state.messages; let assistantIndex = -1; for (let i = messages.length - 1; i >= 0; i--) { const candidate = messages[i]; if (candidate.role === "assistant" && this.#isSameAssistantMessage(candidate, message)) { assistantIndex = i; break; } } if (assistantIndex < 0) return undefined; const unresolvedToolCallIds = new Set(resolvedToolCallIds); for (let i = assistantIndex + 1; i < messages.length; i++) { const candidate = messages[i]; if (candidate.role === "toolResult") unresolvedToolCallIds.delete(candidate.toolCallId); } if (unresolvedToolCallIds.size > 0) return undefined; return reasonlessAbort ? "reasonless-abort" : "stream-stall"; } /** * Mid-generation transport death: the local idle watchdog's stream stall, an * HTTP/2 stream reset, a gateway close without the terminal event, or a * truncated stream envelope. A premature close (no finish_reason/terminal * event) and an envelope that never reached its terminal frame — including a * gateway's in-band `error` frame, which the provider reports as the same * envelope truncation — are the same failure class as a stall or reset. * Resets and closes are ignored while a deliberate abort, disposal, or * streaming-edit abort owns the stream. */ #isMidStreamTransportFailure(message: AssistantMessage, id: number): boolean { if (message.stopReason !== "error" || !AIError.retriable(id)) return false; const errorMessage = message.errorMessage ?? ""; if (STREAM_STALL_ERROR_RE.test(errorMessage)) return true; if (this.#host.abortInProgress() || this.#host.isDisposed() || this.#host.streamingEditAbortTriggered()) { return false; } return ( AIError.isStreamEnvelopeTruncationText(errorMessage) || HTTP2_STREAM_RESET_ERROR_RE.test(errorMessage) || AIError.PYTHON_HTTP2_STREAM_RESET_PATTERN.test(errorMessage) || AIError.PYTHON_HTTP_INCOMPLETE_CHUNK_PATTERN.test(errorMessage) || PREMATURE_STREAM_CLOSE_ERROR_RE.test(errorMessage) ); } /** * Retried turns remove the failed assistant message from active context. * Thinking-only partials are safe to discard and replay: reasoning models * routinely stall after long thinking with no visible output, and duplicated * thinking display is materially lower harm than duplicated final text. * Whitespace-only and buffered text are likewise safe since nothing meaningful * reached the user. Committed text, generated images, server tools, and retained * tool calls are NOT safe: each has already rendered or may have side effects, * so replaying the turn can duplicate user-visible output or work. */ #hasReplayUnsafeOutput(message: AssistantMessage): boolean { return message.content.some( block => block.type === "toolCall" || block.type === "image" || block.type === "anthropicServerTool" || (block.type === "text" && this.#host.textOutputCommitted() && block.text.trim().length > 0), ); } /** * Known provider routes can repeatedly close after completing an expensive * reasoning phase. One retry covers a transient edge failure; the normal * ten-retry budget would otherwise replay the same reasoning cycle unchanged. */ #isBoundedThinkingStreamClose(message: AssistantMessage): boolean { if (!message.content.some(block => block.type === "thinking" && block.thinking.trim().length > 0)) { return false; } const errorMessage = message.errorMessage ?? ""; return ( (message.provider === "openrouter" && /server_error:\s*stream closed with reason:\s*error/i.test(errorMessage)) || (message.provider === "github-copilot" && message.model === "grok-4.6" && message.api === "openai-responses" && /OpenAI responses stream closed before a terminal response event was received/i.test(errorMessage)) ); } /** * First-attempt mid-stream socket drop with streamed progress: the transport * died after the model had already emitted reasoning or tool calls, so the * failure says nothing about model health — retry the same model once before * consulting the fallback chain. A drop with no streamed content keeps the * immediate fallback (a different route may genuinely help), as do later * attempts. Never applies once the retry budget is spent (e.g. * `retry.maxRetries: 0`): there is no same-model retry left, so the * fallback-chain consult is the only recovery. Mirrors the stall handler's * socket check, extended to turns with tool calls. */ #isFirstAttemptMidStreamSocketDrop(message: AssistantMessage, id: number, retryBudgetExhausted: boolean): boolean { if (this.#retryAttempt !== 1 || retryBudgetExhausted) return false; if (message.stopReason !== "error" || !AIError.retriable(id)) return false; if (this.#host.streamingEditAbortTriggered()) return false; if (!isUnexpectedSocketCloseMessage(message.errorMessage ?? "")) return false; return message.content.some( block => (block.type === "thinking" && block.thinking.trim().length > 0) || block.type === "toolCall" || (block.type === "text" && block.text.trim().length > 0), ); } /** Checks whether a provider error represents a classifier refusal. */ isClassifierRefusal(message: AssistantMessage): boolean { if (message.stopReason !== "error") return false; const stopType = message.stopDetails?.type; return stopType === "refusal" || stopType === "sensitive"; } #getRetryFallbackResolutionContext(): RetryFallbackResolutionContext { return { chains: this.#getRetryFallbackChains(), getModelRole: role => this.#host.settings.getModelRole(role), modelLookup: this.#host.modelRegistry, }; } #getRetryFallbackChains(): RetryFallbackChains { return getRetryFallbackChains(this.#host.settings); } /** * Validate configured fallback chains, logging each warning and appending it * to `configWarnings`. Runs once: at construction unless deferred by the owner. */ validateRetryFallbackChains(): void { if (this.#fallbackChainsValidated) return; this.#fallbackChainsValidated = true; let deferred = false; validateRetryFallbackChains( this.#host.settings, this.#host.modelRegistry, message => { logger.warn(message); this.#fallbackChainWarnings.add(message); this.#host.configWarnings.push(message); }, { isDiscoveryPending: provider => { const pending = this.#host.modelRegistry.isProviderDiscoveryPending(provider); if (pending) deferred = true; return pending; }, }, ); this.#pendingDiscoveryDeferredValidation = deferred; } /** Whether startup fallback-chain validation deferred any warning pending in-flight discovery (#10048). */ hasPendingDiscoveryDeferredFallbackValidation(): boolean { return this.#pendingDiscoveryDeferredValidation; } /** * Re-run fallback-chain validation once background discovery has settled and * reconcile `configWarnings`. Startup validation suppresses "unknown model" * warnings for selectors whose config-declared discovery provider had not yet * populated the registry (a cold cache after `omp update` bumps the discovery * namespace, #10048). With discovery done, drop any startup warning discovery * resolved and surface warnings for selectors that stayed unknown. * * @returns true when `configWarnings` changed and the header must rebuild. */ revalidateRetryFallbackChainsAfterDiscovery(): boolean { const definitive = new Set(); validateRetryFallbackChains(this.#host.settings, this.#host.modelRegistry, message => definitive.add(message)); this.#pendingDiscoveryDeferredValidation = false; let changed = false; for (const message of Array.from(this.#fallbackChainWarnings)) { if (definitive.has(message)) continue; const index = this.#host.configWarnings.indexOf(message); if (index !== -1) { this.#host.configWarnings.splice(index, 1); changed = true; } this.#fallbackChainWarnings.delete(message); } for (const message of definitive) { if (this.#fallbackChainWarnings.has(message)) continue; logger.warn(message); this.#fallbackChainWarnings.add(message); this.#host.configWarnings.push(message); changed = true; } return changed; } #getRetryFallbackRevertPolicy(): RetryFallbackRevertPolicy { return getRetryFallbackRevertPolicy(this.#host.settings); } /** Clears fallback ownership after an explicit model change or a restore. */ clearActiveRetryFallback(): void { this.#activeRetryFallback = undefined; this.#fallbackRoutedFor = undefined; } /** Checks whether a fallback selector remains in cooldown. */ isRetryFallbackSelectorSuppressed(selector: RetryFallbackSelector): boolean { return this.#host.modelRegistry.isSelectorSuppressed(selector.raw); } /** Records the cooldown that should suppress a failing selector. */ noteRetryFallbackCooldown(currentSelector: string, retryAfterMs: number | undefined, errorMessage: string): void { let cooldownMs = retryAfterMs; if (!cooldownMs || cooldownMs <= 0) { const reason = parseRateLimitReason(errorMessage); cooldownMs = reason === "UNKNOWN" ? 5 * 60 * 1000 : calculateRateLimitBackoffMs(reason); } this.#host.modelRegistry.suppressSelector(currentSelector, Date.now() + cooldownMs); } /** * Map the failing model selector to the chain key that owns it, by * specificity: an exact model-selector key, then a `provider/*` wildcard, * then a model role whose current assignment matches, then `default`. * Model-oriented keys win over roles so a chain follows the model across * role reassignments. */ resolveRetryFallbackRole( currentSelector: string, currentModel: Model | null | undefined = this.#host.model(), roleHint?: string, ): string | undefined { return resolveRetryFallbackChainKey( this.#getRetryFallbackResolutionContext(), currentSelector, currentModel, roleHint ?? this.#liveRetryRoleHint(currentModel), ); } /** * Chain keys to consult for the active model, most specific walk first: the * chain that owns the current fallback walk, then the chain the CURRENT model * owns when that is a different key. * * The second key is what makes a chain reachable from the end of another one. * `#activeRetryFallback.role` is pinned at the first hop and never * re-resolved, so a session that lands on the last entry of one chain has no * candidate left and retries the same model until the budget is gone — even * when that entry is itself a chain key with its own fallbacks configured * (`anthropic/claude-opus-5 -> … -> runinfra/deepseek-v4-pro`, whose own * `runinfra/deepseek-v4-pro -> openrouter/deepseek/…` chain was unreachable). * * Two keys is the whole walk per attempt. Chains that point at each other * alternate models instead of looping in place, and every hop still spends a * retry attempt, so the budget terminates either way. */ retryFallbackChainKeys( currentSelector: string, currentModel: Model | null | undefined = this.#host.model(), options?: { pinnedRole?: string; roleHint?: string }, ): string[] { const pinned = options?.pinnedRole ?? this.#activeRetryFallback?.role; const current = this.resolveRetryFallbackRole(currentSelector, currentModel, options?.roleHint); if (!pinned) return current ? [current] : []; return current && current !== pinned ? [pinned, current] : [pinned]; } /** * Live session role for chain lookup, provided its assignment still matches the active model. * Temporary switches record the `fallback` role; skipping them keeps the assigned role (e.g. a * subagent's own, possibly empty, chain) once the session is back on that role's model. */ #liveRetryRoleHint(currentModel: Model | null | undefined): string | undefined { const role = this.#host.sessionManager?.getLastModelChangeRole?.({ skipEphemeral: true }); if (!role || !currentModel) return undefined; const configured = this.#host.settings.getModelRole(role); if (!configured) return undefined; const resolved = resolveModelOverride([configured], this.#host.modelRegistry, this.#host.settings); return resolved.model && modelsAreEqual(resolved.model, currentModel) ? role : undefined; } /** Finds fallback candidates that follow the active selector. */ findRetryFallbackCandidates( role: string, currentSelector: string, currentModel: Model | null | undefined = this.#host.model(), options?: { wrapAround?: boolean }, ): RetryFallbackSelector[] { return findRetryFallbackCandidates( this.#getRetryFallbackResolutionContext(), role, currentSelector, currentModel, options, ); } async #maybeApplyUsageAwareFallback(signal: AbortSignal, confirmer?: UsageFallbackConfirmer): Promise { if (!cfgRetryUsageAwareFallback.get(this.#host.settings)) return false; const currentModel = this.#host.model(); if (!currentModel) return false; const sessionId = this.#host.sessionManager.getSessionId(); const currentSelector = formatRetryFallbackSelector(currentModel, this.#host.thinkingLevel()); let health: ModelUsageHealth; try { health = await this.#host.modelRegistry.authStorage.health.model(currentModel.provider, { modelId: currentModel.id, sessionId: this.#host.sessionId(), baseUrl: currentModel.baseUrl, reserveFraction: cfgRetryUsageReservePct.get(this.#host.settings) / 100, signal, }); } catch (error) { if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false; logger.debug("Usage-aware runtime preflight failed open", { provider: currentModel.provider, model: currentModel.id, error: String(error), }); return false; } if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false; const selectedAccount = health.accounts.find(account => account.selected); if (health.state === "healthy") { // A healthy sibling does not end the selected account's reserve episode. if (selectedAccount?.state === "healthy") this.#usageReserveApproval = undefined; if ( selectedAccount && selectedAccount.state !== "healthy" && health.accounts.some(account => account.state === "healthy") ) { this.#host.modelRegistry.authStorage.sessions.release(currentModel.provider, this.#host.sessionId()); } return false; } // Missing quota data is not evidence that the reserve episode ended. if (health.state === "unknown") return false; if (health.state !== "reserve") this.#usageReserveApproval = undefined; const reservePolicy = cfgRetryUsageReservePolicy.get(this.#host.settings); if (reservePolicy === "fail-closed") { const condition = health.state === "reserve" ? "reserve reached" : "usage depleted"; throw new Error( `${USAGE_PREFLIGHT_BLOCKED_PREFIX} ${condition} for ${currentSelector}; reserve policy is fail-closed.`, ); } if ( reservePolicy === "confirm" && health.state === "reserve" && this.#usageReserveApproval?.sessionId === sessionId && this.#usageReserveApproval.model === formatModelStringWithRouting(currentModel) ) { return false; } if (!cfgRetryModelFallback.get(this.#host.settings) || this.#host.modelFallbackDisabled?.()) return false; const fallback = await this.#findKeyedRetryFallback(currentSelector, currentModel, signal); if (!fallback) return false; let shouldFallback = health.state === "depleted" || reservePolicy === "auto" || !confirmer; if (!shouldFallback && health.state === "reserve" && confirmer) { const remainingFraction = selectedAccount?.remainingFraction ?? health.accounts.reduce((minimum, account) => { if (account.remainingFraction === undefined) return minimum; return minimum === undefined ? account.remainingFraction : Math.min(minimum, account.remainingFraction); }, undefined); shouldFallback = await this.#confirmUsageFallback( confirmer, { from: currentSelector, to: fallback.selector.raw, remainingPercent: remainingFraction === undefined ? undefined : Math.max(0, remainingFraction * 100), }, signal, ); if ( signal.aborted || this.#host.sessionManager.getSessionId() !== sessionId || !modelsAreEqual(this.#host.model(), currentModel) ) return false; } if (!shouldFallback) { // Auto thinking changes effort between turns, not the coding-plan quota. this.#usageReserveApproval = { model: formatModelStringWithRouting(currentModel), sessionId }; return false; } this.#usageReserveApproval = undefined; return this.applyRetryFallbackCandidate(fallback.role, fallback.selector, currentSelector, { pinFallback: true, apiKey: fallback.apiKey, signal, reason: describeUsageFallback(health, cfgRetryUsageReservePct.get(this.#host.settings)), }); } /** * Switches off a model whose provider has no usable credential onto the first * keyed fallback candidate. Unpinned, so the primary is restored once its * credential returns. */ async applyMissingCredentialFallback(signal: AbortSignal): Promise { if (!cfgRetryModelFallback.get(this.#host.settings) || this.#host.modelFallbackDisabled?.()) return false; const currentModel = this.#host.model(); if (!currentModel) return false; const currentSelector = formatRetryFallbackSelector(currentModel, this.#host.thinkingLevel()); const fallback = await this.#findKeyedRetryFallback(currentSelector, currentModel, signal); if (!fallback) return false; return this.applyRetryFallbackCandidate(fallback.role, fallback.selector, currentSelector, { apiKey: fallback.apiKey, signal, reason: `no credential for ${currentModel.provider}`, }); } /** First keyed, eligible fallback candidate; undefined when none or when the walk went stale. */ async #findKeyedRetryFallback( currentSelector: string, currentModel: Model, signal: AbortSignal, ): Promise<{ role: string; selector: RetryFallbackSelector; apiKey: string } | undefined> { const stale = () => signal.aborted || !modelsAreEqual(this.#host.model(), currentModel); const ceiling = this.#host.thinkingLevelCeiling(); for (const role of this.retryFallbackChainKeys(currentSelector, currentModel)) { for (const candidate of this.findRetryFallbackCandidates(role, currentSelector, currentModel)) { if (this.isRetryFallbackSelectorSuppressed(candidate)) continue; const resolved = resolveModelOverride([candidate.raw], this.#host.modelRegistry, this.#host.settings); const candidateModel = resolved.model ?? this.#host.modelRegistry.find(candidate.provider, candidate.id); if (!candidateModel || !this.#host.modelRegistry.hasConfiguredAuth(candidateModel)) continue; if (ceiling !== undefined && !modelSupportsEffortCeiling(candidateModel, ceiling)) continue; // Skip a candidate whose window cannot hold the live context so we // never switch onto an oversized request (issue #8065). if (!this.#host.contextFitsModel(candidateModel)) continue; try { const candidateHealth = await this.#host.modelRegistry.authStorage.health.model( candidateModel.provider, { modelId: candidateModel.id, sessionId: this.#host.sessionId(), baseUrl: candidateModel.baseUrl, reserveFraction: cfgRetryUsageReservePct.get(this.#host.settings) / 100, signal, }, ); if (stale()) return undefined; if (candidateHealth.state === "depleted" || candidateHealth.state === "reserve") continue; if (candidateHealth.state === "healthy") { const selected = candidateHealth.accounts.find(account => account.selected); if ( selected && selected.state !== "healthy" && candidateHealth.accounts.some(account => account.state === "healthy") ) { this.#host.modelRegistry.authStorage.sessions.release( candidateModel.provider, this.#host.sessionId(), ); } } } catch { if (stale()) return undefined; // Unknown usage fails open for an otherwise valid fallback. } if (stale()) return undefined; let apiKey: string | undefined; try { apiKey = await this.#host.modelRegistry.getApiKey(candidateModel, this.#host.sessionId(), { signal }); } catch { if (stale()) return undefined; continue; } if (stale()) return undefined; if (!apiKey) continue; return { role, selector: candidate, apiKey }; } } return undefined; } async #confirmUsageFallback( confirmer: UsageFallbackConfirmer, confirmation: UsageFallbackConfirmation, signal: AbortSignal, ): Promise { if (signal.aborted) return false; const aborted = Promise.withResolvers(); const onAbort = () => aborted.resolve(false); signal.addEventListener("abort", onAbort, { once: true }); try { return await Promise.race([confirmer(confirmation, signal), aborted.promise]); } finally { signal.removeEventListener("abort", onAbort); } } /** * Whether applying `candidate` at `selector`'s thinking level would leave the * request unchanged: the same routed model identity (provider, id AND the * `@upstream` route `formatModelStringWithRouting` preserves) and the same * effective thinking level after the ceiling clamp in * {@link applyRetryFallbackCandidate} and the per-model clamp in * `setThinkingLevel`. A different route or a different effective level is a * real change of request and stays eligible. */ #isNoOpRetryFallback(candidate: Model, selector: RetryFallbackSelector): boolean { const active = this.#host.model(); if (!active || formatModelStringWithRouting(candidate) !== formatModelStringWithRouting(active)) return false; const configured = this.#host.configuredThinkingLevel(); const requested = selector.thinkingLevel ?? configured; if (requested === AUTO_THINKING || configured === AUTO_THINKING) return requested === configured; const effective = resolveThinkingLevelForModel( candidate, clampThinkingLevelToCeiling(candidate, requested, this.#host.thinkingLevelCeiling()), ); return effective === configured; } async applyRetryFallbackCandidate( role: string, selector: RetryFallbackSelector, currentSelector: string, options?: { pinFallback?: boolean; apiKey?: string; signal?: AbortSignal; reason?: string }, ): Promise { const resolved = resolveModelOverride([selector.raw], this.#host.modelRegistry, this.#host.settings); const candidate = resolved.model ?? this.#host.modelRegistry.find(selector.provider, selector.id); if (!candidate) { throw new Error(`Retry fallback model not found: ${selector.raw}`); } const apiKey = options?.apiKey ?? (await this.#host.modelRegistry.getApiKey(candidate, this.#host.sessionId(), { signal: options?.signal })); if (!apiKey) { throw new Error(`No API key for retry fallback ${selector.raw}`); } if (options?.signal?.aborted) return false; // Capture the configured selector (auto-aware) so a fallback chain preserves // `auto` instead of collapsing it to the level it resolved to this turn. const currentThinkingLevel = this.#host.configuredThinkingLevel(); const requestedThinkingLevel = selector.thinkingLevel ?? currentThinkingLevel; // A fallback selector's explicit level (or the carried level after the // replacement model's floor clamp) must never exceed the session's // per-spawn effort ceiling. const nextThinkingLevel = requestedThinkingLevel === AUTO_THINKING ? requestedThinkingLevel : clampThinkingLevelToCeiling(candidate, requestedThinkingLevel, this.#host.thinkingLevelCeiling()); const candidateSelector = formatModelStringWithRouting(candidate); const previousModel = this.#host.model(); // Capture the edit mode under the outgoing model so the base system prompt // can be re-synced once the swap commits — `edit.modelVariants` and other // model-dependent prompt policy would otherwise stay pinned to the // chain-head model after an auto-fallback (issue #11983). const previousEditMode = this.#host.resolveActiveEditMode(); // Mark routing BEFORE the swap: `setModelWithProviderSessionReset` moves the // model and fans `model_changed` out to subscribers synchronously, and a // listener reading attribution in that window must already see the incoming // candidate as fallback-routed. Attribution itself is safe regardless — it // names the last model that served, which this swap has not changed. const routedBeforeSwap = this.#fallbackRoutedFor; const servedBeforeSwap = this.#activeRetryFallback?.served; this.#markFallbackRouted(); if (this.#activeRetryFallback) this.#activeRetryFallback.served = false; await this.#host.setModelWithProviderSessionReset(candidate); if (options?.signal?.aborted) { this.#fallbackRoutedFor = routedBeforeSwap; if (this.#activeRetryFallback) this.#activeRetryFallback.served = servedBeforeSwap; if (previousModel && this.#host.model() === candidate) { await this.#host.setModelWithProviderSessionReset(previousModel); } return false; } if (this.#host.model() !== candidate) { this.#fallbackRoutedFor = routedBeforeSwap; if (this.#activeRetryFallback) this.#activeRetryFallback.served = servedBeforeSwap; return false; } this.#host.sessionManager.appendModelChange(candidateSelector, EPHEMERAL_MODEL_CHANGE_ROLE, true); this.#host.settings.getStorage()?.recordModelUsage(candidateSelector); this.#host.setThinkingLevel(nextThinkingLevel); if (!this.#activeRetryFallback) { this.#activeRetryFallback = { role, originalSelector: currentSelector, originalThinkingLevel: currentThinkingLevel, lastAppliedFallbackThinkingLevel: nextThinkingLevel, pinned: options?.pinFallback === true, }; } else { this.#activeRetryFallback.lastAppliedFallbackThinkingLevel = nextThinkingLevel; this.#activeRetryFallback.pinned = this.#activeRetryFallback.pinned || options?.pinFallback === true; } await this.#host.syncAfterModelChange(previousEditMode, { retryFallback: true }); await this.#host.emitSessionEvent({ type: "retry_fallback_applied", from: currentSelector, to: selector.raw, role, reason: options?.reason, }); return true; } async #tryRetryModelFallback( currentSelector: string, failedMessage: AssistantMessage, options?: { excludeProvider?: string; pinFallback?: boolean; preserveFailedTurn?: boolean; wrapAround?: boolean; /** This hop is a classifier-refusal walk, which visits each model at most once per turn. */ refusalWalk?: boolean; /** The retry policy retries the failed model if this hop does not switch. */ sameModelRetryAvailable?: boolean; }, ): Promise { const ceiling = this.#host.thinkingLevelCeiling(); const latestAssistant = options?.preserveFailedTurn ? failedMessage : this.#host.agent.state.messages.findLast( (message): message is AssistantMessage => message.role === "assistant" && message !== failedMessage, ); // Permitted redemption targets are catalog policy on the refused model. const creditHandle = failedMessage.fallbackCreditHandle; const failedModel = creditHandle ? this.#host.modelRegistry.find(failedMessage.provider, failedMessage.model) : undefined; const creditTargets = failedModel ? fallbackCreditTargets(failedModel) : []; // Deferring a sibling Anthropic candidate below is only worth it while // another attempt on the model that signed the replayed thinking follows: // the active model is that model, and the retry policy will retry it. Any // other failure — on a model that did not sign the thinking, or one the // policy does not retry (a refusal, a policy denial, a non-retryable error, // a wait past the delay cap, a spent budget) — has no source model to go // back to, so deferring left the whole chain untried and ended the turn. // Under retry.rateLimitWalksChain a rate limit on the signer counts as no // attempt too: the retry re-hits the same cap, while a chain entry may sit // under a different one. const sourceModel = this.#host.model(); const signerRetryRemains = options?.sameModelRetryAvailable === true && latestAssistant !== undefined && !!sourceModel && sourceModel.provider === latestAssistant.provider && sourceModel.id === latestAssistant.model; for (const role of this.retryFallbackChainKeys(currentSelector)) { for (const selector of this.findRetryFallbackCandidates(role, currentSelector, undefined, options)) { if (this.isRetryFallbackSelectorSuppressed(selector)) continue; // A refusal walk visits any one model at most once per turn. Without this a // pair of chains naming each other alternates: each hop is a model the walk // has already been refused on, and the budget no longer stops it. if (options?.refusalWalk && this.#refusalWalkTried.has(formatRetryFallbackBaseSelector(selector))) continue; const resolved = resolveModelOverride([selector.raw], this.#host.modelRegistry, this.#host.settings); const candidate = resolved.model ?? this.#host.modelRegistry.find(selector.provider, selector.id); if (!candidate) continue; // A candidate that would leave the request exactly as it is — same // routed model, same effective thinking level — is not a switch, and // must never be applied: `findRetryFallbackCandidates` excludes the // current model by selector STRING, while what gets applied is what // `resolveModelOverride` resolves that string to, clamped. When those // disagree, the swap "succeeds" onto the failing request, the caller // sets `switchedModel`, and the retry budget is reset to 1 on every // failure — an unbounded retry loop against a model that cannot work. if (this.#isNoOpRetryFallback(candidate, selector)) continue; if (options?.excludeProvider === candidate.provider) continue; // Redeeming Anthropic fallback credit replays the refused request's frozen // body; the server validates thinking blocks across the boundary itself // using the credit token. const canRedeemFallbackCredit = creditHandle !== undefined && candidate.api === "anthropic-messages" && candidate.provider === failedMessage.provider && creditTargets.includes(candidate.id) && Date.now() < creditHandle.expiresAt; // Anthropic signatures and redacted blocks are model-bound, while the // latest assistant response must remain byte-identical. While another // attempt on the model that signed them is still allowed, prefer it to a // same-provider model switch, which satisfies neither constraint; a later // cross-provider candidate's message transform can demote the foreign // thinking safely. A refusal, an account-policy denial, a non-retryable // error, a wait past `retry.maxDelayMs` or a spent budget makes no such // attempt, and, when `retry.rateLimitWalksChain` is on, a rate limit on // the signer makes only a futile one, so the chain moves to the // candidate. // A classifier refusal is one of those: the source model declines the same // request again, and Anthropic's documented retry for a refusal resends // the history unchanged on another Claude model, which drops the thinking // blocks it cannot read (preserved thinking, "Switching models // mid-conversation"). // While the refusal's fallback credit can still be redeemed, only its // targets pass, so an earlier non-target entry cannot forfeit the credit. // An Anthropic-compatible proxy that will not verify a sibling model's // signature is covered by the transport, not by skipping the hop: the // `Invalid signature in thinking block` 400 escalates to a retry that // replays no thinking and succeeds, and the session stops replaying it // afterwards. const creditLive = creditHandle !== undefined && creditTargets.length > 0 && Date.now() < creditHandle.expiresAt; const refusalKeepsCredit = options?.refusalWalk === true && creditLive; if ( (signerRetryRemains || refusalKeepsCredit) && !canRedeemFallbackCredit && candidate.api === "anthropic-messages" && latestAssistant?.api === "anthropic-messages" && latestAssistant.provider === candidate.provider && latestAssistant.model !== candidate.id && latestAssistant.content.some( block => (block.type === "thinking" && Boolean(block.thinkingSignature?.trim())) || block.type === "redactedThinking", ) ) { continue; } // A candidate whose effort floor exceeds the per-spawn ceiling would be // clamped UP past the cap by its model floor — skip it entirely. if (ceiling !== undefined && !modelSupportsEffortCeiling(candidate, ceiling)) continue; // Skip a candidate whose window cannot hold the retry context. The // failed assistant is excluded only when retry removes it; preserved // unexecuted-tool turns remain part of the request (issue #8065). if (!this.#host.contextFitsModel(candidate, options?.preserveFailedTurn ? undefined : failedMessage)) { continue; } // A session restricted to an OAuth account pool throws for a provider // none of its pooled accounts can serve; that candidate is unavailable, // exactly like one without a credential. let apiKey: string | undefined; try { apiKey = await this.#host.modelRegistry.getApiKey(candidate, this.#host.sessionId()); } catch (error) { if (!(error instanceof AIError.MissingApiKeyError)) throw error; } if (!apiKey) continue; const previousEditMode = this.#host.resolveActiveEditMode(); const previousSystemPrompt = [...this.#host.agent.state.systemPrompt]; const previousTools = [...this.#host.agent.state.tools]; const applied = await this.applyRetryFallbackCandidate(role, selector, currentSelector, { ...options, reason: `Request failed: ${failedMessage.errorMessage ?? "provider returned an error without details"}`, }); // Redemption replays the refused request's frozen `system` and `tools`. If the // swap rebuilt either, the fallback model would be prompted with, and call tools // through, a surface the agent no longer runs: send a fresh request instead. const { systemPrompt, tools } = this.#host.agent.state; const requestSurfaceUnchanged = this.#host.resolveActiveEditMode() === previousEditMode && Bun.deepEquals(systemPrompt, previousSystemPrompt) && tools.length === previousTools.length && tools.every((tool, index) => tool === previousTools[index]); if ( applied && options?.pinFallback === true && creditHandle !== undefined && canRedeemFallbackCredit && requestSurfaceUnchanged ) { this.#activeFallbackCreditRedemption = { targetSelector: `${candidate.provider}/${candidate.id}`, handle: creditHandle, }; } return applied; } } return false; } /** The active model when it is a Fireworks Fast (`-fast`) variant, else undefined. */ #activeFireworksFastModel(): Model | undefined { const model = this.#host.model(); return model?.provider === "fireworks" && isFireworksFastModelId(model.id) ? model : undefined; } /** * True when the current turn failed on a Fireworks Fast (`-fast`) model in a * way that should degrade to the reliable base (Standard) model. Fast is a * speed-optimized router with no SLA, so any *pre-content* failure — a * transient overload/5xx or a hard "router/model not found / unsupported" — * is worth retrying on the base id. Skips failures the base model shares: * context overflow (compaction's job), usage limits and auth errors (same * account/key), and turns that already emitted any replay-unsafe output. * Requires the base model to exist in the registry. */ isFireworksFastFallbackEligible(message: AssistantMessage): boolean { const model = this.#activeFireworksFastModel(); if (!model) return false; if (message.stopReason !== "error") return false; if (this.#isUsagePreflightBlocked(message)) return false; if (this.#hasReplayUnsafeOutput(message)) return false; // A content refusal/sensitivity stop is the model's decision, not a route // failure — switching to the base model would just re-trigger it. if (this.isClassifierRefusal(message)) return false; const id = this.#classifyRetryMessage(message); if (AIError.isContextOverflow(message, model.contextWindow ?? 0)) return false; if (AIError.is(id, AIError.Flag.UsageLimit)) return false; if (AIError.is(id, AIError.Flag.AuthFailed)) return false; // A thinking loop is a same-model resample signal, not a router fault, so a // base-model swap would abandon the loop-guard redirect (issue #8760). if (AIError.is(id, AIError.Flag.ThinkingLoop)) return false; // Byte/media budget 413s fail identically on the base model — swapping cannot fix them (#9235). if (AIError.isPayloadRejection(message)) return false; return this.#host.modelRegistry.find("fireworks", toFireworksBaseModelId(model.id)) !== undefined; } /** * True when a turn failed with a hard (non-retryable) provider error but a * configured `retry.fallbackChains` entry covers the active model: the same * model is not worth retrying, yet a DIFFERENT model is a fresh chance, so * the chain is consulted before the error becomes final. Skips failures a * model switch cannot fix or must not replay: cancellations (abort-flavored * errors are not model faults), context overflow (compaction's job), * classifier refusals (chain consult is handled on the retryable path with * `pinFallback`), and turns that already emitted replay-unsafe output. */ isHardErrorFallbackEligible(message: AssistantMessage): boolean { if (message.stopReason !== "error") return false; if (this.#isUsagePreflightBlocked(message)) return false; const model = this.#host.model(); if (!model) return false; const immutableAnthropicThinkingError = model.api === "anthropic-messages" && (message.errorStatus === 400 || message.errorId === 400 || message.errorMessage?.startsWith("400 ") === true) && AIError.isImmutableLatestThinkingText(message.errorMessage ?? ""); if (immutableAnthropicThinkingError) return false; const retrySettings = cfgRetry.get(this.#host.settings); if (!retrySettings.enabled || !retrySettings.modelFallback) return false; if (this.isClassifierRefusal(message)) return false; const id = this.#classifyRetryMessage(message); if (AIError.is(id, AIError.Flag.Abort) || AIError.is(id, AIError.Flag.UserInterrupt)) return false; if (!retryFallbackAdmits(retrySettings.fallbackOn, AIError.is(id, AIError.Flag.UsageLimit))) return false; // Text-ambiguous overflows waive the veto; usage-backed do not — see AIError.isTextAmbiguousContextOverflow (#9235). const contextWindow = model.contextWindow ?? 0; const textAmbiguousOverflow = AIError.isTextAmbiguousContextOverflow(id, message, contextWindow); if (!textAmbiguousOverflow && AIError.isContextOverflow(message, contextWindow)) { return false; } if (this.#hasReplayUnsafeOutput(message)) return false; const currentSelector = formatRetryFallbackSelector(model, this.#host.thinkingLevel()); return this.retryFallbackChainKeys(currentSelector).some( role => this.findRetryFallbackCandidates(role, currentSelector).length > 0, ); } /** * Switch the active model from a Fireworks Fast (`-fast`) variant to its base * (Standard) id and stick there for the rest of the session — the auto * fallback that makes Fast a safe default. Returns false when the current * model is not a fast variant, the base id is missing, or it has no key. */ async #tryFireworksFastFallback(currentSelector: string): Promise { const model = this.#activeFireworksFastModel(); if (!model) return false; const baseModel = this.#host.modelRegistry.find("fireworks", toFireworksBaseModelId(model.id)); if (!baseModel) return false; const apiKey = await this.#host.modelRegistry.getApiKey(baseModel, this.#host.sessionId()); if (!apiKey) return false; const baseSelector = formatModelStringWithRouting(baseModel); const previousEditMode = this.#host.resolveActiveEditMode(); // A capability degrade is fallback routing too, even though it arms no // chain: the base model must not be reported as the configured primary. this.#markFallbackRouted(); await this.#host.setModelWithProviderSessionReset(baseModel); this.#host.sessionManager.appendModelChange(baseSelector, EPHEMERAL_MODEL_CHANGE_ROLE, true); this.#host.settings.getStorage()?.recordModelUsage(baseSelector); await this.#host.syncAfterModelChange(previousEditMode, { retryFallback: true }); await this.#host.emitSessionEvent({ type: "retry_fallback_applied", from: currentSelector, to: baseSelector, role: "fireworks-fast", reason: "Request rejected by the Fast tier. Retrying on the Standard tier.", }); return true; } async #maybeRestoreRetryFallbackPrimary(): Promise { if (!this.#activeRetryFallback) return false; if (this.#activeRetryFallback.pinned) return false; if (this.#getRetryFallbackRevertPolicy() !== "cooldown-expiry") return false; const { originalSelector: originalSelectorRaw, originalThinkingLevel, lastAppliedFallbackThinkingLevel, } = this.#activeRetryFallback; const originalSelector = parseRetryFallbackSelector(originalSelectorRaw, this.#host.modelRegistry); if (!originalSelector) { // Defensive: the stored selector is always produced by // `formatRetryFallbackSelector`, so it should never fail to parse. If it // somehow does, nothing is restored and the session keeps running on the // fallback — so drop the chain record but NOT `#fallbackRouted`, whose // clearing would report the fallback's remaining turns as the primary. this.#activeRetryFallback = undefined; return false; } const currentModel = this.#host.model(); if (!currentModel) return false; const currentSelector = formatRetryFallbackSelector(currentModel, this.#host.thinkingLevel()); if (currentSelector === originalSelector.raw) { if (!this.isRetryFallbackSelectorSuppressed(originalSelector)) { this.clearActiveRetryFallback(); } return false; } if (this.isRetryFallbackSelectorSuppressed(originalSelector)) return false; const resolvedPrimary = resolveModelOverride( [originalSelector.raw], this.#host.modelRegistry, this.#host.settings, ); const primaryModel = resolvedPrimary.model ?? this.#host.modelRegistry.find(originalSelector.provider, originalSelector.id); if (!primaryModel) return false; const apiKey = await this.#host.modelRegistry.getApiKey(primaryModel, this.#host.sessionId()); if (!apiKey) return false; const currentThinkingLevel = this.#host.configuredThinkingLevel(); const thinkingToApply = currentThinkingLevel === lastAppliedFallbackThinkingLevel ? originalThinkingLevel : currentThinkingLevel; const primarySelector = formatModelStringWithRouting(primaryModel); const previousEditMode = this.#host.resolveActiveEditMode(); // Clear before the swap: `setModelWithProviderSessionReset` and // `setThinkingLevel` both notify subscribers, and an observer reading // attribution in that window would see the restored primary still tagged // as fallback-served. this.clearActiveRetryFallback(); await this.#host.setModelWithProviderSessionReset(primaryModel); this.#host.sessionManager.appendModelChange(primarySelector, EPHEMERAL_MODEL_CHANGE_ROLE); this.#host.settings.getStorage()?.recordModelUsage(primarySelector); this.#host.setThinkingLevel(thinkingToApply); // The plain comparison: a hop kept the primary's prompt, so this is a no-op, unless a rebuild // while on the fallback rendered the fallback's policy, which the primary must not keep. await this.#host.syncAfterModelChange(previousEditMode); return true; } #parseRetryAfterMsFromError(errorMessage: string): number | undefined { // Single call into the provider-aware parser: it merges every supported // form (account resets, retry-after-ms, legacy retry-after / // x-ratelimit-reset text) by longest-wins, so a shorter reset phrase // can never shadow a longer retry signal carried in the same message. return extractProviderRetryHint(this.#host.model()?.provider, errorMessage); } /** * Handle retryable errors with exponential backoff, credential rotation, and * model-fallback chains. Also entered for NON-retryable errors when a switch * is the recovery (`fireworksFastFallback`, `hardErrorFallback`): then a * successful model switch retries immediately, and a failed switch surfaces * the error without a same-model backoff retry. * @returns true if retry was initiated, false if max retries exceeded or disabled */ async #handleRetryableError(message: AssistantMessage, options?: RetryPolicyOptions): Promise { try { return await this.#applyRetryPolicy(message, options); } catch (error) { // The policy's dead ends close their own saga through // #closeRetrySaga, so a write that throws there has already closed // it. A throw anywhere else on the way, such as recording an attempt // before its retry is scheduled, leaves the saga open, and nothing // else would close it. if (this.#retryPromise) { await this.#closeRetrySaga( { success: false, attempt: this.#retryAttempt, finalError: message.errorMessage }, {}, ); } throw error; } } /** * The retry policy {@link #handleRetryableError} applies. A throw outside * its dead ends leaves the saga for the caller to close. */ async #applyRetryPolicy(message: AssistantMessage, options?: RetryPolicyOptions): Promise { const retrySettings = cfgRetry.get(this.#host.settings); if (this.#host.abortInProgress() || this.#host.isDisposed()) return false; // The Fireworks Fast→base degrade is an intrinsic model-selection safety net, // not a retry loop, so it runs even when the user disabled retries: it switches // the model once and lets the base turn proceed. if (!retrySettings.enabled && !options?.fireworksFastFallback) return false; const classifierRefusal = this.isClassifierRefusal(message); const generation = this.#host.promptGeneration(); if (this.#retryAttempt === 0) this.#genericAbortStreak = 0; this.#retryAttempt++; this.#genericAbortStreak = message.errorMessage !== undefined && Object.hasOwn(GENERIC_ABORT_MESSAGES, message.errorMessage) ? this.#genericAbortStreak + 1 : 0; // Create retry promise on first attempt so waitForRetry() can await it // Ensure only one promise exists (avoid orphaned promises from concurrent calls) if (!this.#retryPromise) { const { promise, resolve } = Promise.withResolvers(); this.#retryPromise = promise; this.#retryResolve = resolve; } // All attempts on the current model are spent. Don't fail yet: the // fallback chain below gets one last consult. Credential rotation can // consume the entire budget without the fallback branch ever running // (every rotation sets switchedCredential and skips it), so without // this last resort a provider-wide usage cap never fails over to the // configured chain. const maxRetries = this.#isBoundedThinkingStreamClose(message) ? Math.min(retrySettings.maxRetries, 1) : retrySettings.maxRetries; const retryBudgetExhausted = this.#retryAttempt > maxRetries || this.#genericAbortStreak > GENERIC_ABORT_MAX_RETRIES; const errorMessage = message.errorMessage || "Unknown error"; const id = this.#classifyRetryMessage(message); const requestProvider = (message as IssuedProviderAssistantMessage)[kUsageLimitRequestProvider]; const oldProviderLimitFor = (model: Model | undefined): boolean => requestProvider !== undefined && model !== undefined && AIError.is(id, AIError.Flag.UsageLimit) && requestProvider !== model.provider; const preserveFailedTurn = options?.preserveFailedTurn === true || ((classifierRefusal || AIError.is(id, AIError.Flag.MalformedFunctionCall) || AIError.retriable(id)) && this.#unexecutedToolCallsReplaySafe(message)); const rateLimitReason = parseRateLimitReason(errorMessage); // A rate limit is a cap the next attempt on this model re-hits, so // retry.rateLimitWalksChain lets it skip the signed-thinking deferral in // #tryRetryModelFallback and walk the chain at once. A chain entry may sit // under a different cap; when it shares this one, its own failure walks the // chain on. Off (the default) the deferral holds for a rate limit too. const sourceRateLimited = retrySettings.rateLimitWalksChain && (message.errorStatus === 429 || rateLimitReason === "RATE_LIMIT_EXCEEDED"); const staleOpenAIResponsesReplayError = AIError.is(id, AIError.Flag.StaleResponsesItem); const accountPolicyDenial = AIError.is(id, AIError.Flag.AccountPolicy); let recordedUsageLimitOutcome = await this.#usageLimitOutcomes.get(message); let staleUsageLimit = oldProviderLimitFor(this.#host.model()); if (staleUsageLimit) recordedUsageLimitOutcome = undefined; let parsedRetryAfterMs = staleUsageLimit ? undefined : this.#parseRetryAfterMsFromError(errorMessage); let delayMs = staleOpenAIResponsesReplayError ? 0 : calculateRetryBackoffDelayMs(retrySettings.baseDelayMs, this.#retryAttempt); // Transient rate/concurrency caps stay on the same credential, but must // honor their reason-specific windows. The default exponential base // (≈500ms, capped at 8s) otherwise re-hits the cap and burns the retry // budget before either window can clear. An explicit provider // retry-after is authoritative in both directions, so the heuristic // window only applies when the error carries no parsed timing. if ( !staleOpenAIResponsesReplayError && !AIError.is(id, AIError.Flag.UsageLimit) && parsedRetryAfterMs === undefined && (rateLimitReason === "CONCURRENT_LIMIT" || rateLimitReason === "RATE_LIMIT_EXCEEDED" || (rateLimitReason === "MODEL_CAPACITY_EXHAUSTED" && (AIError.isUsageLimitStatus(message.errorStatus) || /\b(?:429|MODEL_CAPACITY_EXHAUSTED|MODEL_CAPACITY_EXCEEDED|resource.?exhausted)\b/i.test( errorMessage, )))) ) { const reasonBackoffMs = calculateRateLimitBackoffMs(rateLimitReason); if (reasonBackoffMs > delayMs) delayMs = reasonBackoffMs; } let switchedCredential = false; let switchedModel = false; // Set when a usage-limit error pinned the wait to credential // availability — suppresses the generic retry-after bump below. let usageLimitWaitMs: number | undefined; let siblingAvailabilityWaitMs = recordedUsageLimitOutcome?.retryAtMs === undefined ? undefined : Math.max(0, recordedUsageLimitOutcome.retryAtMs - Date.now()) + SIBLING_UNBLOCK_BUFFER_MS; if (staleOpenAIResponsesReplayError) { this.#host.resetCurrentResponsesProviderSession("stale replay error"); } // Rotation can spend the same-route retry budget before the last account // hits its quota. Past the budget only a confirmed reset may continue: a // rotation that keeps claiming a usable sibling would otherwise retry forever. if (!staleOpenAIResponsesReplayError && recordedUsageLimitOutcome) { const rotated = recordedUsageLimitOutcome.switchedCredential && !retryBudgetExhausted; let restored = false; if (!rotated) { const resetAbortController = new AbortController(); this.#retryAbortController?.abort(); this.#retryAbortController = resetAbortController; const startedAtMs = Date.now(); const unblockAtMs = parsedRetryAfterMs === undefined ? undefined : startedAtMs + parsedRetryAfterMs; try { for (let attempt = 0; ; attempt++) { const result = await this.#host.maybeAutoRedeemReset(unblockAtMs); resetAbortController.signal.throwIfAborted(); if (result.restored) { restored = true; break; } if (result.retryAfterMs === undefined || attempt >= maxRetries) break; // Only retry the safe eligibility read, never the reset mutation or // model request. Respect provider pacing within the configured wait cap. const readDelayMs = Math.max( 1_000, result.retryAfterMs, calculateRetryBackoffDelayMs(retrySettings.baseDelayMs, attempt + 1), ); if (retrySettings.maxDelayMs > 0 && Date.now() - startedAtMs + readDelayMs > retrySettings.maxDelayMs) break; await sleepLong(readDelayMs, resetAbortController.signal); } } catch (error) { if (!resetAbortController.signal.aborted) throw error; if (this.#retryAbortController !== resetAbortController) return false; return this.#endCancelledRetry(); } finally { if (this.#retryAbortController === resetAbortController) this.#retryAbortController = undefined; } } if (rotated || restored) { switchedCredential = true; delayMs = 0; } else { // No sibling credential is usable right now. Wait for whichever // comes first: the current account's actual unblock deadline, or // the earliest moment a temporarily blocked sibling frees up // (e.g. a 60s post-401 block or a 5-min usage-probe block) — the // next attempt's getApiKey re-ranks and picks it up. Without the // sibling minimum, one short-lived sibling block escalates a // recoverable situation into the provider's multi-hour wait and // trips the fail-fast cap below. // The deadline merges every independent provider signal by // longest-wins (mirroring extractRetryHint): the error-text hint // (or heuristic fallback when hintless), the credential block // the mark call persisted, and — when the usage report is a // complete authority — its own reset, which replaces the // heuristic instead of sleeping a guess. usageLimitWaitMs = recordedUsageLimitOutcome.retryAfterMs; if (recordedUsageLimitOutcome.reportResetAtMs !== undefined && parsedRetryAfterMs === undefined) { // A hintless error can still carry an authoritative // usage-report window: it replaces THIS call's 30-minute // heuristic guess in both directions — sleeping the guess // past a shorter reported reset overshoots, and vice // versa. usageLimitWaitMs = Math.max(0, recordedUsageLimitOutcome.reportResetAtMs - Date.now()); } if ( recordedUsageLimitOutcome.priorBlockedUntilMs !== undefined && recordedUsageLimitOutcome.priorBlockedUntilTimed === true ) { // The merged deadline below masks a pre-existing block // shorter than this call's heuristic fallback (longest-wins // in the mark). The prior deadline is that block's own // provenance — an earlier response's provider-stated // window — and must survive this call's heuristic guess: // waking before it retries a still-blocked credential. A // prior deadline that was itself only a heuristic guess // carries no such authority and must not extend the wait // past an authoritative report window. const priorRemainingMs = Math.max(0, recordedUsageLimitOutcome.priorBlockedUntilMs - Date.now()); if (priorRemainingMs > usageLimitWaitMs) usageLimitWaitMs = priorRemainingMs; } if (recordedUsageLimitOutcome.blockedUntilMs !== undefined) { // The stored deadline merges every mark call for this // credential (longest-wins). Only a deadline past what // THIS call requested — a longer report window, or a // longer block an earlier sibling-session response stored // for the shared credential — may override the wait: // retrying before the credential's actual unblock time // re-hits the cap, but this call's own heuristic // contribution must not re-inflate over the authoritative // report window above. const requestedBlockedUntilMs = Date.now() + (recordedUsageLimitOutcome.retryAfterMs ?? 0); if (recordedUsageLimitOutcome.blockedUntilMs > requestedBlockedUntilMs) { const blockedRemainingMs = Math.max(0, recordedUsageLimitOutcome.blockedUntilMs - Date.now()); if (blockedRemainingMs > usageLimitWaitMs) usageLimitWaitMs = blockedRemainingMs; } } if (siblingAvailabilityWaitMs !== undefined && siblingAvailabilityWaitMs < usageLimitWaitMs) { usageLimitWaitMs = siblingAvailabilityWaitMs; } if (usageLimitWaitMs > delayMs) { delayMs = usageLimitWaitMs; } } } const currentModel = this.#host.model(); // The model may have changed while the quota reset lookup awaited. if (!staleUsageLimit && oldProviderLimitFor(currentModel)) { staleUsageLimit = true; recordedUsageLimitOutcome = undefined; parsedRetryAfterMs = undefined; usageLimitWaitMs = undefined; siblingAvailabilityWaitMs = undefined; switchedCredential = false; delayMs = calculateRetryBackoffDelayMs(retrySettings.baseDelayMs, this.#retryAttempt); } const allowModelFallback = options?.allowModelFallback !== false && !this.#host.modelFallbackDisabled?.(); const currentSelector = currentModel ? formatRetryFallbackSelector(currentModel, this.#host.thinkingLevel()) : undefined; if (accountPolicyDenial && currentModel) { const accountPolicyKey = `${currentModel.provider}/${currentModel.id}`; const rotationsRemaining = this.#accountPolicyRotationsRemaining.get(accountPolicyKey) ?? Math.max(0, this.#host.modelRegistry.authStorage.credentials.list(currentModel.provider).length - 1); this.#accountPolicyRotationsRemaining.set(accountPolicyKey, Math.max(0, rotationsRemaining - 1)); // `rotate` may sleep out a sibling's short block; register its signal // as the retry controller so `abortRetry()` cancels that wait exactly // like the backoff sleep below. const rotateAbortController = new AbortController(); this.#retryAbortController?.abort(); this.#retryAbortController = rotateAbortController; try { switchedCredential = ( await this.#host.modelRegistry.authStorage.limits.rotate(currentModel.provider, this.#host.sessionId(), { error: errorMessage, modelId: currentModel.id, signal: rotateAbortController.signal, waitForSibling: rotationsRemaining > 0, }) ).switched; // Rotation is advisory: expired blocks or a repeatedly offered sibling // must not retry beyond one pool traversal in this saga. switchedCredential &&= rotationsRemaining > 0; } catch (error) { if (!rotateAbortController.signal.aborted) throw error; if (this.#retryAbortController !== rotateAbortController) return false; return this.#endCancelledRetry(); } finally { if (this.#retryAbortController === rotateAbortController) this.#retryAbortController = undefined; } if (switchedCredential) delayMs = 0; } // A thinking-loop abort is not a provider failure — it is the loop guard // asking for a same-model resample, paired with a hidden // `thinking-loop-redirect` notice that only makes sense on the model that // looped. Walking `fallbackChains` (or parking the selector on a cooldown) // would swap a healthy planning turn to another family based on chain // contents, not model health (issue #8760). Keep it on the same model; the // retry budget still bounds a genuinely stuck stream. const thinkingLoop = AIError.is(id, AIError.Flag.ThinkingLoop); // A Codex steering rejection answers our own `response.steer`, not the // model's health, and the provider already stopped steering the session: // replay on the same model while same-model retries remain. Once the // budget is spent, the chain keeps its last-resort consult. const sameModelSteerReplay = !retryBudgetExhausted && AIError.isCodexSteerRejection({ api: currentModel?.api, errorMessage }); const effectiveUsageLimitWaitMs = usageLimitWaitMs ?? (siblingAvailabilityWaitMs === undefined ? (recordedUsageLimitOutcome?.retryAfterMs ?? parsedRetryAfterMs) : Math.min( recordedUsageLimitOutcome?.retryAfterMs ?? parsedRetryAfterMs ?? Infinity, siblingAvailabilityWaitMs, )); const waitForSiblingCredential = siblingAvailabilityWaitMs !== undefined && effectiveUsageLimitWaitMs !== undefined && effectiveUsageLimitWaitMs <= retrySettings.maxDelayMs; const longUsageLimitFallback = currentModel !== undefined && resolveModelPolicy(currentModel).catalog.longUsageLimitFallback === true && retrySettings.maxDelayMs > 0 && effectiveUsageLimitWaitMs !== undefined && effectiveUsageLimitWaitMs > retrySettings.maxDelayMs && /\bGoUsageLimitError\b/.test(errorMessage) && (!this.#hasReplayUnsafeOutput(message) || this.#unexecutedToolCallsReplaySafe(message)); // Fail-fast cap: if the provider asks us to wait longer than // retry.maxDelayMs and we have no fallback credential or model to // switch to, surface the error instead of sleeping. Defends against // 3-hour Anthropic rate-limit windows that would otherwise leave a // subagent (or interactive session) silently hung. The original // assistant error message is preserved in agent state so the caller // can act on it. // Opt-out: retry.waitForUsageReset lets a provider-stated usage-limit // reset (Flag.UsageLimit — 5h/weekly quota windows, CN 使用上限, spend // caps, … on any provider) sleep past the cap. Gated on authoritative // provider timing: either a parsed reset hint from the error text, or // a complete usage-report window (every exhausted limit carries a // future reset). Usage-limit errors with neither fall back to the // 30-minute QUOTA_EXHAUSTED heuristic, and sleeping on that for a // permanent error (402 balance, dead spend cap) would hold the session // through repeated heuristic sleeps instead of surfacing it. Bounded // by the stated wait so an unrelated large backoff cannot sneak // through. const maxDelayMs = retrySettings.maxDelayMs; // An account out of money has no reset to wait for: the QUOTA_EXHAUSTED // heuristic backoff is a guess at a clock that does not exist, and a // `retry.maxDelayMs` set at or above it (the 30-minute default ceiling // users raise) would sleep through the whole guess. Surface the provider // error instead, so the user tops up and the error frame's Retry // re-attempts at once rather than idling in a retry the session cannot // finish. const billingCap = isBillingCapUsageLimit(message.errorStatus, errorMessage); const exceedsDelayCap = (waitMs: number): boolean => { if (billingCap && !staleUsageLimit) return true; const waitForUsageReset = retrySettings.waitForUsageReset === true && recordedUsageLimitOutcome !== undefined && (parsedRetryAfterMs !== undefined || recordedUsageLimitOutcome.reportResetAtMs !== undefined) && effectiveUsageLimitWaitMs !== undefined && waitMs <= effectiveUsageLimitWaitMs; return maxDelayMs > 0 && waitMs > maxDelayMs && !waitForUsageReset; }; if (!staleOpenAIResponsesReplayError && !switchedCredential && currentSelector) { // The retry budget bounds same-model retries against a failing provider. A // classifier refusal is not that: retrying the same model reproduces it, and // the next model in the chain is one request away and often answers. Stopping // the walk at the budget ends the turn with models left untried. // // The budget was also what terminated the walk: `retryFallbackChainKeys` // consults the current model's own chain as well as the pinned one, so chains // that name each other alternate rather than loop in place, and every hop used // to spend an attempt. The walk is bounded here instead — a refusal visits any // one model at most once per turn, so A -> B -> A terminates whatever the // chains say. const refusalWalkKey = this.#refusalWalkKey(currentSelector); const refusalWalkRepeats = classifierRefusal && this.#refusalWalkTried.has(refusalWalkKey); if ( allowModelFallback && retrySettings.modelFallback && retryFallbackAdmits(retrySettings.fallbackOn, AIError.is(id, AIError.Flag.UsageLimit)) && !staleUsageLimit && !thinkingLoop && !sameModelSteerReplay && !waitForSiblingCredential && !refusalWalkRepeats && !this.#isFirstAttemptMidStreamSocketDrop(message, id, retryBudgetExhausted) ) { if (classifierRefusal) { this.#refusalWalkTried.add(refusalWalkKey); } else { // A usage-limit wait already knows when this provider can serve // the session again (report reset, merged credential block, // sibling unblock); cooling down for less sends the revert back // to a still-exhausted primary. A switched credential means a // sibling is free now, and that wait covers only the spent one. const usageCooldownMs = recordedUsageLimitOutcome?.switchedCredential ? undefined : usageLimitWaitMs; this.noteRetryFallbackCooldown(currentSelector, usageCooldownMs ?? parsedRetryAfterMs, errorMessage); } switchedModel = await this.#tryRetryModelFallback(currentSelector, message, { excludeProvider: longUsageLimitFallback ? currentModel.provider : undefined, pinFallback: classifierRefusal, refusalWalk: classifierRefusal, preserveFailedTurn, wrapAround: longUsageLimitFallback, // Deferring a chain candidate only pays when the code below makes // another attempt on this model that can succeed. Each term is a // branch below that, with no switch, ends the turn instead (a spent // budget, a refusal or account-policy denial, a hard-error or // Fast-degrade consult the classifier would not retry, a wait past // the fail-fast cap), or, under `retry.rateLimitWalksChain`, a rate // limit on this model, which the next attempt re-hits. sameModelRetryAvailable: !retryBudgetExhausted && !sourceRateLimited && !classifierRefusal && !accountPolicyDenial && !( (options?.fireworksFastFallback || options?.hardErrorFallback) && !this.isRetryableError(message) ) && !exceedsDelayCap( usageLimitWaitMs === undefined && parsedRetryAfterMs && parsedRetryAfterMs > delayMs ? parsedRetryAfterMs : delayMs, ), }); } // Auto fallback from a Fireworks Fast variant to its base model. Independent // of the role-fallback setting: it's intrinsic to the Fast contract (speed // best-effort, degrade to Standard on failure) and triggers on hard router // errors the generic retry classifier would otherwise reject. if (!switchedModel && allowModelFallback && options?.fireworksFastFallback) { switchedModel = await this.#tryFireworksFastFallback(currentSelector); } if (switchedModel) { delayMs = 0; } else if (usageLimitWaitMs === undefined && parsedRetryAfterMs && parsedRetryAfterMs > delayMs) { delayMs = parsedRetryAfterMs; } } if (retryBudgetExhausted && staleUsageLimit) { // The spent attempts and their backoff belong to the replaced provider. // Give the selected model its first request without inheriting the old // provider's fail-fast cap or reset window. this.#retryAttempt = 1; delayMs = 0; } else if (retryBudgetExhausted) { if (!switchedModel && !switchedCredential) { const attempt = this.#retryAttempt - 1; message.errorMessage = `Retry budget exhausted after ${attempt} ${attempt === 1 ? "retry" : "retries"}: ${errorMessage}`; await this.#closeRetrySaga( { success: false, attempt, finalError: errorMessage }, { terminalTurn: message, retryErrors: { status: "superseded" } }, ); return false; } // A fallback model gets a fresh retry budget. Credential rotation // instead keeps the cumulative attempt count while bypassing the // same-route budget: every distinct account must be tried first. if (switchedModel) { this.#retryAttempt = 1; this.#genericAbortStreak = 0; } } if ((classifierRefusal || accountPolicyDenial) && !switchedCredential && !switchedModel) { // A prior attempt in this saga already announced `auto_retry_start` // (retryAttempt was incremented for each call to this method, so > 1 // means at least one earlier attempt started the loop) but this // attempt is not going to retry — the saga must close with its own // `auto_retry_end` so subscribers tracking retry-outstanding state // (e.g. suppressing a duplicate error toast) don't stay latched on // an announcement that never resolves. if (this.#retryAttempt > 1) { await this.#closeRetrySaga( { success: false, attempt: this.#retryAttempt - 1, finalError: errorMessage }, { terminalTurn: message }, ); } this.#retryAttempt = 0; this.resolveRetry(); return false; } // A fallback switch was the whole reason we entered (Fast→base degrade or // a hard-error chain consult) but it could not happen (e.g. no candidate // has a credential). Don't fall through to backing-off and retrying the // failing model for an error the generic classifier wouldn't retry — // surface it instead. if ( (options?.fireworksFastFallback || options?.hardErrorFallback) && !switchedModel && !this.isRetryableError(message) ) { // Same auto_retry_end backstop as the classifier-refusal branch above. if (this.#retryAttempt > 1) { await this.#closeRetrySaga( { success: false, attempt: this.#retryAttempt - 1, finalError: errorMessage }, { terminalTurn: message }, ); } this.#retryAttempt = 0; this.resolveRetry(); return false; } // The fail-fast cap (`exceedsDelayCap`, defined above the chain consult). if (exceedsDelayCap(delayMs) && !switchedCredential && !switchedModel) { const attempt = this.#retryAttempt; this.#retryAttempt = 0; await this.#closeRetrySaga( { success: false, attempt, // A billing cap has no wait to report — the account is out of // money — so the message the user acts on is the provider's own. finalError: billingCap ? errorMessage : `Provider requested ${Math.ceil(delayMs)}ms wait, exceeds retry.maxDelayMs (${maxDelayMs}ms). Original error: ${errorMessage}`, }, { terminalTurn: message }, ); return false; } await this.#recordPendingRetryError(message, id, { switchedCredential, switchedModel, delayMs }); // Quota resets elapse while the machine sleeps; capture the deadline before event delivery. const usageResetDeadline = AIError.is(id, AIError.Flag.UsageLimit) ? Date.now() + delayMs : undefined; await this.#host.emitSessionEvent({ type: "auto_retry_start", attempt: this.#retryAttempt, maxAttempts: maxRetries, delayMs, errorMessage, errorId: message.errorId, }); if (this.#host.abortInProgress() || this.#host.promptGeneration() !== generation) { return this.#endCancelledRetry(); } // Resolved stream-stall tools and proven-unexecuted malformed/refused // calls keep their assistant/result pair. Continuation then sees explicit // synthetic results and cannot repeat a side effect. if (!preserveFailedTurn) { this.removeAssistantMessageFromActiveContext(message, "auto-retry"); } // A thinking/response loop retried into identical context loops again. Inject a // hidden redirect so the retried turn sees a directive to break the repeated // pattern instead of re-sampling the same stalled reasoning. this.#maybeInjectThinkingLoopRedirect(id); this.#maybeInjectMalformedFunctionCallRedirect(id, maxRetries, preserveFailedTurn, switchedModel); // Quota waits follow the wall clock; ordinary backoff stays monotonic. const retryAbortController = new AbortController(); this.#retryAbortController?.abort(); this.#retryAbortController = retryAbortController; try { if (usageResetDeadline !== undefined) { await sleepUntil(usageResetDeadline, retryAbortController.signal); } else { await sleepLong(delayMs, retryAbortController.signal); } } catch { if (this.#retryAbortController !== retryAbortController) { return false; } // Aborted during sleep - emit end event so UI can clean up return this.#endCancelledRetry(); } if (this.#retryAbortController === retryAbortController) { this.#retryAbortController = undefined; } // The identity-keyed removal above can miss when a context rebuild // recreated the failed turn's message object between settle and retry // (fresh identity, same failed tail — issue #5382). Agent.continue() // rejects any assistant tail, so a missed removal fails the scheduled // retry locally before a provider request is ever made. Re-check the // tail after the backoff (covering rebuilds during the sleep too) and // strip a still-failed assistant tail by position. Never when preserving // the failed turn — the kept turn ends in synthetic tool results that // continue() accepts — and never once a newer prompt owns the session. if (!preserveFailedTurn && this.#host.promptGeneration() === generation) { this.#stripFailedAssistantTail(); } // Retry via continue() outside the agent_end event callback chain. A // continuation that still fails locally must close the retry saga — // otherwise auto_retry_end never fires, retryPromise stays pending, and // the in-flight prompt() (and the TUI retry indicator) hang forever. this.#host.scheduleAgentContinue({ source: "automatic-retry", delayMs: 1, generation, shouldContinue: () => this.#retryAttempt > 0, onError: error => void this.#failRetryAfterLocalContinueError(message, error).catch(failure => logger.error("Closing a retry saga after a failed local continuation threw", { error: failure instanceof Error ? failure.message : String(failure), }), ), }); return true; } /** Closes a retry saga whose credential wait or backoff sleep was aborted. */ async #endCancelledRetry(): Promise { const attempt = this.#retryAttempt; this.#retryAttempt = 0; this.#retryAbortController = undefined; await this.#host.emitSessionEvent({ type: "auto_retry_end", success: false, attempt, finalError: "Retry cancelled", }); this.#clearPendingRetryErrors(); this.resolveRetry(); return false; } /** * Positional backstop for {@link removeAssistantMessageFromActiveContext}: * when the identity check missed, the failed assistant turn is still the * active tail and the scheduled continue() would reject it. An * error/aborted-stopped assistant tail is never legal continuation input * and no recovery path wants it replayed on the wire, so drop it by * position; any healthy tail is left untouched. */ #stripFailedAssistantTail(): void { const messages = this.#host.agent.state.messages; const tail = messages[messages.length - 1]; if (tail?.role !== "assistant") return; if (tail.stopReason !== "error" && tail.stopReason !== "aborted") return; logger.debug("agent active context failed assistant tail stripped positionally", { stopReason: tail.stopReason, timestamp: tail.timestamp, }); this.#host.agent.replaceMessages(messages.slice(0, -1)); } /** * Close the retry saga when the scheduled continue() failed locally (no * provider request was made). Mirrors the other retry dead-ends: emit the * closing `auto_retry_end` so subscribers stop showing retry progress, and * resolve the retry promise so the in-flight prompt() unwinds (issue #5382). */ async #failRetryAfterLocalContinueError(message: AssistantMessage, error: unknown): Promise { if (this.#retryAttempt === 0) return; const attempt = this.#retryAttempt; this.#retryAttempt = 0; const localError = error instanceof Error ? error.message : String(error); await this.#closeRetrySaga( { success: false, attempt, finalError: `Retry continuation failed locally: ${localError}. Original error: ${message.errorMessage ?? "Unknown error"}`, }, { terminalTurn: message }, ); } /** * Inject a hidden redirect notice when a thinking/response loop is being retried, so * the retried turn carries an instruction to break the repeated pattern instead of * re-sampling the same stalled context. Injected on every {@link AIError.Flag.ThinkingLoop} * retry (the failed assistant is dropped each attempt, so the notice does not accumulate * unboundedly). No-op unless `id` carries the ThinkingLoop flag and the loop guard is * enabled. The notice is generic on purpose — the detector's detail can quote raw model * text, which must not be interpolated into a higher-priority developer message. */ #maybeInjectThinkingLoopRedirect(id: number): void { if (!AIError.is(id, AIError.Flag.ThinkingLoop)) return; if (cfgModelLoopGuardEnabled.get(this.#host.settings) !== true) return; this.#host.agent.appendMessage({ role: "custom", customType: THINKING_LOOP_REDIRECT_TYPE, content: thinkingLoopRedirectTemplate, display: false, attribution: "agent", timestamp: Date.now(), }); this.#host.sessionManager.appendCustomMessageEntry( THINKING_LOOP_REDIRECT_TYPE, thinkingLoopRedirectTemplate, false, undefined, "agent", ); } #maybeInjectMalformedFunctionCallRedirect( id: number, maxRetries: number, preserveFailedTurn: boolean, switchedModel: boolean, ): void { if (!AIError.is(id, AIError.Flag.MalformedFunctionCall)) return; if (preserveFailedTurn || switchedModel) return; const reminder = prompt.render(malformedFunctionCallRetryTemplate, { retryCount: this.#retryAttempt, maxRetries, }); this.#host.agent.appendMessage({ role: "developer", content: [{ type: "text", text: reminder }], attribution: "agent", timestamp: Date.now(), }); } /** * Cancel in-progress retry. */ abortRetry(): void { this.#retryAbortController?.abort(); // Note: _retryAttempt is reset in the catch block of _autoRetry this.resolveRetry(); } async #promptAgentWithIdleRetry( messages: AgentMessage[], options?: { toolChoice?: ToolChoice }, onClaimed?: () => void, ): Promise { const deadline = Date.now() + 30_000; for (;;) { // A claim whose synchronous prefix throws (busy agent, missing // model) returns an already-rejected promise. Probe for that // before firing `onClaimed`: a busy-retry parked behind an // autonomous run must not report that run's streaming as ours // going live. The rejection handler rethrows, so the error // still surfaces at the await below. let claimFailed = false; try { const claim = this.#host.agent.prompt(messages, options).then(undefined, error => { claimFailed = true; throw error; }); // One microtask turn: an already-rejected promise's reaction // job is queued ahead of this await's continuation, so // `claimFailed` is settled by the time we resume. await Promise.resolve(); if (!claimFailed) onClaimed?.(); await claim; return; } catch (err) { if (!(err instanceof AgentBusyError)) { throw err; } if (Date.now() >= deadline) { throw new Error("Timed out waiting for prior agent run to finish before prompting."); } await this.#host.agent.waitForIdle(); } } } /** Whether auto-retry is currently in progress */ get isRetrying(): boolean { return this.#retryPromise !== undefined; } /** Whether auto-retry is enabled */ get autoRetryEnabled(): boolean { return cfgRetryEnabled.get(this.#host.settings); } /** * Toggle auto-retry. When `persist` is false (the default) the change is a * session-scoped runtime override rather than a durable global write, so the * `set_auto_retry` RPC command configures only its own session instead of * mutating the machine-global `config.yml`. */ setAutoRetryEnabled(enabled: boolean, persist = false): void { if (persist) { cfgRetryEnabled.set(this.#host.settings, enabled); cfgRetryEnabled.clearOverride(this.#host.settings); } else { cfgRetryEnabled.override(this.#host.settings, enabled); } } /** * Whether the transcript tail is a failed/aborted assistant turn whose most * recent activity was a tool call: synthetic placeholder results trail the * failed turn (stream died mid-tool-call), or the message right before the * aborted boundary is an errored/aborted tool result (Esc landed during * tool execution). Drives the TUI's idle "F5 to Retry" status row; * {@link retry} is the matching action. */ get hasAbortedToolCallTail(): boolean { const active = this.#host.agent.state.messages; if (abortedToolCallTail(active)) return true; // A trailing assistant message is authoritative for a live session: a // settled successful turn leaves nothing to retry. if (active[conversationEnd(active) - 1]?.role === "assistant") return false; // A restored session omits the failed turn (and its synthetic results) // from provider context, so — mirroring retry() — the persisted display // transcript decides whether a retryable tool-call tail exists. return abortedToolCallTail(this.#host.sessionManager.buildSessionContext({ transcript: true }).messages); } /** * Manually retry the last failed assistant turn. * Removes the error message from active agent state when present and * re-attempts with a fresh retry budget. * * A stream that stalls or aborts mid-tool-call ends the turn with * `stopReason: "error" | "aborted"` and then appends one synthetic * {@link isSyntheticToolResultMessage tool_result} per emitted tool call to * preserve the provider's tool_use/tool_result pairing (see * `createAbortedToolResult` in `agent-loop.ts`). Those placeholders trail the * failed assistant turn, so the retry lookback walks back over them before * checking the assistant message; it strips both the placeholders and the * failed turn before re-attempting. * * A restored session deliberately omits failed assistant turns from provider * context. In that case, the persisted display transcript remains the source * of truth for whether the current branch has a retryable failed tail. * * When the failure is a fully failed/aborted tool batch with complete * arguments (see {@link toolReplayStart}), the retry strips the failed * results (and the aborted boundary) instead of the whole turn, leaving the * tool-calling assistant as the tail — the continuation then re-executes * the same tool calls directly rather than paying a model call to re-issue * them. * * @returns true if retry was initiated, false if no failed turn to retry or agent is busy */ async retry(): Promise { if (this.#host.isStreaming() || this.#host.isCompacting() || this.isRetrying) return false; const messages = this.#host.agent.state.messages; const activeTurnEnd = retryableAssistantTurnEnd(messages); if (activeTurnEnd !== undefined) { const replayStart = toolReplayStart(messages); if (replayStart !== undefined) { await this.#stripForToolReplay(messages, replayStart); } else { // Remove the failed/aborted assistant message plus its synthetic tool // results (same as auto-retry does before re-attempting), keeping the // trailing context deliveries the continuation is sent with. this.#host.agent.replaceMessages([ ...messages.slice(0, activeTurnEnd - 1), ...messages.slice(conversationEnd(messages)), ]); } } else { // A restored session already dropped the failed assistant turn (and its // paired synthetic tool results) from provider context, so the persisted // display transcript is the source of truth for a retryable failed tail. const transcriptMessages = this.#host.sessionManager.buildSessionContext({ transcript: true }).messages; if (retryableAssistantTurnEnd(transcriptMessages) === undefined) return false; // The boundary is already gone from active context; when the intact // failed batch is still the tail, replay the tools directly. const replayStart = toolReplayStart(messages); if (replayStart !== undefined) await this.#stripForToolReplay(messages, replayStart); } // Reset retry budget for a fresh attempt this.#retryAttempt = 0; // Re-attempt the turn this.#host.scheduleAgentContinue({ source: "manual-retry", delayMs: 1 }); return true; } /** * Ready a bare resume of the prompt the model still owes a reply — an Esc or * a failure cut its turn off before any text or tool call. Drops the trailing * cut-off turns from active context, as {@link retry} drops a failed turn (no * provider replays them anyway), so `Agent.continue()` answers the prompt as is. * * @returns the prompt to resume; undefined while busy, or when an assistant * turn already answers the last prompt. */ prepareResume(): AgentMessage | undefined { if (this.#host.isStreaming() || this.#host.isCompacting() || this.isRetrying) return undefined; const messages = this.#host.agent.state.messages; const end = conversationEnd(messages); let tail = end; while (tail > 0 && isCutOffAssistant(messages[tail - 1])) tail--; const prompt = unansweredPrompt(messages, tail); if (prompt && tail < end) { this.#host.agent.replaceMessages([...messages.slice(0, tail), ...messages.slice(end)]); } return prompt; } /** * Strip the failed tool batch (and any aborted boundary) for a tool replay: * reparent the persisted leaf onto the anchor assistant entry — so the * dropped results cannot resurface on reload or pair duplicate toolCallIds * on replay — then cut the active context to the same point. Trailing eval * snapshots go too: none may sit between the calls and their re-run results. */ async #stripForToolReplay(messages: readonly AgentMessage[], replayStart: number): Promise { const tail = messages[conversationEnd(messages) - 1]; if (tail?.role === "assistant") { try { await this.#host.waitForSessionMessagePersistence(tail); } catch (err) { logger.debug("Tool replay: boundary persistence wait failed", { error: err instanceof Error ? err.message : String(err), }); } } const anchor = messages[replayStart - 1] as AssistantMessage; const branch = this.#host.sessionManager.getBranchView(); const persistedEntryId = this.#host.persistedAssistantEntryId(anchor); const anchorEntry = (persistedEntryId === undefined ? undefined : branch.find( entry => entry.id === persistedEntryId && entry.type === "message" && entry.message.role === "assistant", )) ?? branch.findLast( entry => entry.type === "message" && entry.message.role === "assistant" && this.#isSameAssistantMessage(entry.message as AssistantMessage, anchor), ); if (anchorEntry) { this.#host.withBashBranchTransition(() => { this.#host.sessionManager.branch(anchorEntry.id); }); } this.#host.agent.replaceMessages(messages.slice(0, replayStart)); } }