import { isHardBillingToolHttpError, normalizeToolHttpErrorMessage, type ToolHttpError, } from './tool-http-errors'; import type { ToolExecutionErrorSchemaVersion } from '../tool-execution-error'; export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 2; // A provider may need to finish the original request before it can replay the // result for this key. With no Retry-After signal, these two polls give the // original request ten seconds to finish without repeatedly hammering it. export const TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS = [ 5_000, 5_000, ] as const; export const TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_MAX_ATTEMPTS = TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS.length + 1; export const TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS = 8; export const TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS = 2; export const TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS = 3; export const TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS = 1_000; export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000; export const TOOL_EXECUTE_RETRY_DELAY_MAX_MS = 5_000; export const TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS = 60_000; export const TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE = 'AUTH_SCOPE_CHANGED'; export const TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_CODE = 'CUSTOMER_DB_STORAGE_UNAVAILABLE'; /** * A provider/action-local outcome contract emits this only when a provider has * explicitly said that the same idempotency key is still executing. It is not * a generic HTTP 409 policy. */ export const TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_CODE = 'UPSTREAM_IDEMPOTENCY_IN_PROGRESS'; export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_CODE = 'GATEWAY_INVOCATION_IN_PROGRESS'; export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_MAX_ATTEMPTS = 65; export class ToolExecuteAuthScopeChangedError extends Error { readonly code = TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE; readonly status = 409; constructor(message = 'Tool execution auth scope changed.') { super(message); this.name = 'ToolExecuteAuthScopeChangedError'; } } export type ToolExecuteHttpRetryDecision = { retryable: boolean; attemptCap: number; reason: | 'rate_limit' | 'idempotency_in_progress' | 'gateway_invocation_in_progress' | 'customer_db_storage_unavailable' | 'retry_safe_transient_5xx' | 'unsafe_transient_5xx' | 'hard_billing_error' | 'non_retryable_status'; }; export type ToolExecuteHttpFailureOutcome = ToolExecuteHttpRetryDecision & { error: ToolHttpError; shouldRetry: boolean; isRateLimit: boolean; fate: 'retry' | 'settle_row_failure' | 'fail_run'; retryDelayMs: number; backpressureDelayMs: number | null; chargeRetryBudget: boolean; }; export type ToolExecuteHttpFailureAttemptTracker = { next(input: { toolId: string; status: number; bodyText?: string; transientHttpRetrySafe?: boolean; }): number; }; function parseJsonObject(text: string): Record | null { if (!text.trim()) return null; try { const parsed = JSON.parse(text) as unknown; return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record) : null; } catch { return null; } } function isIdempotencyInProgressResponse(input: { status: number; bodyText: string; }): boolean { if (input.status !== 409) return false; const body = parseJsonObject(input.bodyText); return ( body?.code === TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_CODE && body.error_category === 'idempotency_in_progress' ); } function isGatewayInvocationInProgressResponse(input: { status: number; bodyText: string; }): boolean { return ( input.status === 409 && parseJsonObject(input.bodyText)?.code === TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_CODE ); } function isCustomerDbStorageUnavailableResponse(input: { toolId: string; status: number; bodyText: string; }): boolean { if ( input.status !== 503 || (input.toolId !== 'query_customer_db' && input.toolId !== 'customer_db_query_customer_db') ) { return false; } return ( parseJsonObject(input.bodyText)?.code === TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_CODE ); } function idempotencyInProgressRetryDelayMs(attempt: number): number { return ( TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS[ Math.min( Math.max(0, attempt - 1), TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS.length - 1, ) ] ?? TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS ); } export function parseToolExecuteAuthScopeChangedError(input: { status: number; bodyText: string; }): ToolExecuteAuthScopeChangedError | null { if (input.status !== 409) return null; const body = parseJsonObject(input.bodyText); const code = typeof body?.code === 'string' ? body.code.trim() : ''; if (code !== TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE) return null; const message = typeof body?.error === 'string' && body.error.trim() ? body.error.trim() : typeof body?.message === 'string' && body.message.trim() ? body.message.trim() : undefined; return new ToolExecuteAuthScopeChangedError(message); } function decideToolExecuteHttpRetry(input: { status: number; idempotencyInProgress?: boolean; gatewayInvocationInProgress?: boolean; customerDbStorageUnavailable?: boolean; hardBillingFailure?: boolean; hasRetryAfterHeader?: boolean; transientHttpRetrySafe?: boolean; }): ToolExecuteHttpRetryDecision { if (input.hardBillingFailure) { return { retryable: false, attemptCap: input.status === 429 ? TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS : 1, reason: 'hard_billing_error', }; } if (input.status === 429) { if (!input.hasRetryAfterHeader) { return { retryable: true, attemptCap: TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS, reason: 'rate_limit', }; } return { retryable: true, attemptCap: TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS, reason: 'rate_limit', }; } if (input.gatewayInvocationInProgress) { return { retryable: true, attemptCap: TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_MAX_ATTEMPTS, reason: 'gateway_invocation_in_progress', }; } if (input.idempotencyInProgress) { return { retryable: true, attemptCap: TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_MAX_ATTEMPTS, reason: 'idempotency_in_progress', }; } // This typed response is emitted for a PostgreSQL permission denial. The // failed statement cannot commit, so it is safe to retry even when the SQL // itself is a mutation. Do not generalize this to arbitrary 503 responses. if (input.customerDbStorageUnavailable) { return { retryable: true, attemptCap: TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS, reason: 'customer_db_storage_unavailable', }; } if (input.status >= 500 && input.status < 600) { if (!input.transientHttpRetrySafe) { return { retryable: false, attemptCap: 1, reason: 'unsafe_transient_5xx', }; } return { retryable: true, attemptCap: TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS, reason: 'retry_safe_transient_5xx', }; } return { retryable: false, attemptCap: 1, reason: 'non_retryable_status', }; } export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFailureAttemptTracker { const attemptsByReason: Record< ToolExecuteHttpRetryDecision['reason'], number > = { rate_limit: 0, idempotency_in_progress: 0, gateway_invocation_in_progress: 0, customer_db_storage_unavailable: 0, retry_safe_transient_5xx: 0, unsafe_transient_5xx: 0, hard_billing_error: 0, non_retryable_status: 0, }; return { next(input) { const decision = decideToolExecuteHttpRetry({ status: input.status, idempotencyInProgress: isIdempotencyInProgressResponse({ status: input.status, bodyText: input.bodyText ?? '', }), gatewayInvocationInProgress: isGatewayInvocationInProgressResponse({ status: input.status, bodyText: input.bodyText ?? '', }), customerDbStorageUnavailable: isCustomerDbStorageUnavailableResponse({ toolId: input.toolId, status: input.status, bodyText: input.bodyText ?? '', }), hasRetryAfterHeader: true, transientHttpRetrySafe: input.transientHttpRetrySafe === true, }); attemptsByReason[decision.reason] += 1; return attemptsByReason[decision.reason]; }, }; } export function parseToolExecuteRetryAfterMs( header: string | null | undefined, nowMs = Date.now(), ): number { if (!header) return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS; const seconds = Number(header); if (Number.isFinite(seconds) && seconds > 0) { return Math.ceil(seconds * 1000); } const retryAt = Date.parse(header); if (Number.isFinite(retryAt)) { return Math.max(1, retryAt - nowMs); } return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS; } export function classifyToolExecuteHttpFailure(input: { toolId: string; status: number; attempt: number; bodyText: string; schemaVersion?: ToolExecutionErrorSchemaVersion; retryAfterHeader?: string | null; transientHttpRetrySafe?: boolean; providerLatencyMs?: number; nowMs?: number; }): ToolExecuteHttpFailureOutcome { const transientHttpRetrySafe = input.transientHttpRetrySafe === true; const hasRetryAfterHeader = typeof input.retryAfterHeader === 'string' && input.retryAfterHeader.trim().length > 0; const idempotencyInProgress = isIdempotencyInProgressResponse(input); const gatewayInvocationInProgress = isGatewayInvocationInProgressResponse(input); const customerDbStorageUnavailable = isCustomerDbStorageUnavailableResponse(input); const initialRetryDecision = decideToolExecuteHttpRetry({ status: input.status, idempotencyInProgress, gatewayInvocationInProgress, customerDbStorageUnavailable, hasRetryAfterHeader, transientHttpRetrySafe, }); const retryAfterMs = parseToolExecuteRetryAfterMs( input.retryAfterHeader, input.nowMs, ); const error = normalizeToolHttpErrorMessage({ toolId: input.toolId, status: input.status, attempt: input.attempt, maxAttempts: initialRetryDecision.attemptCap, bodyText: input.bodyText, schemaVersion: input.schemaVersion, retryable: initialRetryDecision.retryable, retryAfterMs: input.status === 429 && hasRetryAfterHeader ? retryAfterMs : null, }); if ( typeof input.providerLatencyMs === 'number' && Number.isFinite(input.providerLatencyMs) && input.providerLatencyMs >= 0 ) { error.message = `${error.message} (provider call ${Math.round(input.providerLatencyMs)}ms)`; } const hardBillingFailure = isHardBillingToolHttpError(error); const retryDecision = decideToolExecuteHttpRetry({ status: input.status, idempotencyInProgress, gatewayInvocationInProgress, customerDbStorageUnavailable, hardBillingFailure, hasRetryAfterHeader, transientHttpRetrySafe, }); const shouldRetry = retryDecision.retryable && input.attempt < retryDecision.attemptCap; // In-attempt retry safety and later-run repairability are different // decisions. A non-idempotent 5xx must not be repeated automatically inside // this execution attempt, but an explicit later `plays run` must be able to // repair that failed work while completed calls remain reusable. Otherwise // one provider gateway failure poisons the semantic call forever. if ( !hardBillingFailure && (retryDecision.retryable || (input.status >= 500 && input.status < 600)) ) { error.receiptFailureKind = 'repairable'; } const retryDelayMs = input.status === 429 ? Math.min( TOOL_EXECUTE_RETRY_DELAY_MAX_MS, Math.max( retryAfterMs, TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt, ), ) : retryDecision.reason === 'idempotency_in_progress' && !hasRetryAfterHeader ? idempotencyInProgressRetryDelayMs(input.attempt) : retryDecision.reason === 'gateway_invocation_in_progress' && !hasRetryAfterHeader ? idempotencyInProgressRetryDelayMs(input.attempt) : retryAfterMs > 0 ? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs) : TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS; return { ...retryDecision, error, shouldRetry, isRateLimit: input.status === 429, fate: shouldRetry ? 'retry' : retryDecision.reason === 'hard_billing_error' ? 'fail_run' : 'settle_row_failure', retryDelayMs, backpressureDelayMs: input.status === 429 && retryDecision.reason !== 'hard_billing_error' ? hasRetryAfterHeader && retryAfterMs > 0 ? retryAfterMs : TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS : null, chargeRetryBudget: shouldRetry && retryDecision.reason !== 'gateway_invocation_in_progress', }; }