import type { WorkReceiptFailureKind } from './work-receipts'; import { LEGACY_TOOL_EXECUTION_ERROR_SCHEMA_VERSION, TOOL_EXECUTION_ERROR_SCHEMA_VERSION, ToolExecutionError, deserializeToolExecutionFailure, isProviderTransientFailure, normalizeToolExecutionCategory, normalizeToolExecutionOrigin, type ToolExecutionErrorSchemaVersion, type ToolExecutionErrorOptions, } from '../tool-execution-error'; const TOOL_HTTP_ERROR_BRAND = Symbol.for('deepline.tool-http-error.v1'); type ToolHttpErrorConstructorInput = { message: string; billing: Record | null; status: number; receiptFailureKind: WorkReceiptFailureKind; options: Partial; }; function applyToolHttpErrorFields( error: { name: string; billing: Record | null; status: number; receiptFailureKind: WorkReceiptFailureKind; }, input: ToolHttpErrorConstructorInput, ): void { error.name = 'ToolHttpError'; error.billing = input.billing; error.status = input.status; error.receiptFailureKind = input.receiptFailureKind; Object.defineProperty(error, TOOL_HTTP_ERROR_BRAND, { configurable: false, enumerable: false, writable: false, value: true, }); } /** * Retained only to deserialize direct SDK callers that explicitly request the * retired schema-0 boundary. Play runtime execution always uses schema 1. */ export class ToolHttpError extends Error { readonly billing: Record | null; /** HTTP status of the failed tool-execute response (e.g. 429, 502). */ readonly status: number; receiptFailureKind: WorkReceiptFailureKind; constructor( message: string, billing: Record | null, status: number, receiptFailureKind: WorkReceiptFailureKind = 'terminal', _options: Partial = {}, ) { super(message); this.billing = billing; this.status = status; this.receiptFailureKind = receiptFailureKind; applyToolHttpErrorFields(this, { message, billing, status, receiptFailureKind, options: _options, }); } static [Symbol.hasInstance](value: unknown): boolean { return Boolean( value && typeof value === 'object' && (value as Record)[TOOL_HTTP_ERROR_BRAND] === true, ); } } class StructuredToolHttpError extends ToolExecutionError { readonly billing: Record | null; readonly status: number; receiptFailureKind: WorkReceiptFailureKind; constructor(input: ToolHttpErrorConstructorInput) { super(input.message, { toolId: input.options.toolId ?? 'unknown_tool', provider: input.options.provider ?? null, operation: input.options.operation ?? null, code: input.options.code ?? null, origin: input.options.origin ?? 'unknown', category: input.options.category ?? 'unknown', retryable: input.options.retryable === true, statusCode: input.options.statusCode === undefined ? input.status : input.options.statusCode, requestId: input.options.requestId ?? null, retryAfterMs: input.options.retryAfterMs ?? null, networkKind: input.options.networkKind ?? null, networkScope: input.options.networkScope ?? null, }); this.billing = input.billing; this.status = input.status; this.receiptFailureKind = input.receiptFailureKind; applyToolHttpErrorFields(this, input); this.name = isProviderTransientFailure(this) ? 'ProviderTransientError' : 'ToolExecutionError'; } } export function createToolHttpError( schemaVersion: ToolExecutionErrorSchemaVersion, message: string, billing: Record | null, status: number, receiptFailureKind: WorkReceiptFailureKind = 'terminal', options: Partial = {}, ): ToolHttpError { if (schemaVersion === LEGACY_TOOL_EXECUTION_ERROR_SCHEMA_VERSION) { return new ToolHttpError( message, billing, status, receiptFailureKind, options, ); } return new StructuredToolHttpError({ message, billing, status, receiptFailureKind, options, }) as ToolHttpError; } function formatCreditAmount(value: unknown): string { if (typeof value !== 'number' || !Number.isFinite(value)) { return String(value ?? '-'); } return Number(value.toFixed(8)).toString(); } function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } function getStringField(value: unknown, key: string): string | null { if (!isRecord(value)) return null; const field = value[key]; return typeof field === 'string' && field.trim() ? field : null; } function getBooleanField(value: unknown, key: string): boolean | null { if (!isRecord(value)) return null; const field = value[key]; return typeof field === 'boolean' ? field : null; } function getObjectField( value: unknown, key: string, ): Record | null { if (!isRecord(value)) return null; const field = value[key]; return isRecord(field) ? field : null; } function isInsufficientCreditsBilling( billing: Record | null, ): billing is Record { return billing?.kind === 'insufficient_credits'; } /** * A transient failure of the Deepline billing PLANE (Convex outage, OCC * exhaustion) surfaced as HTTP 503 `BILLING_UNAVAILABLE` with `retryable: true` * and `failure_description` stating no credits were charged. This is NOT a hard * credit/cap denial: a funded org is temporarily blocked because our billing * store hiccuped, so it must be repairable on the next run rather than cached as * a terminal failure. See `billingUnavailableResponse` in * src/lib/integrations/execute-billing.ts. This class is the exact incident * poison closed by issues #2561/#2706: without this carve-out a billing-plane * outage classifies every affected tool call as `hard_billing_error` -> * `terminal` and the next run replays the cached failure instead of retrying. */ export function isTransientBillingFailurePayload( payload: Record | null, ): boolean { if (!payload) return false; const code = String(payload.code ?? payload.error_code ?? '').toUpperCase(); if (code === 'BILLING_UNAVAILABLE') return true; // Any billing-origin payload that explicitly declares itself retryable is a // billing-infra hiccup, never a hard denial (denials are never retryable). const category = String( payload.error_category ?? payload.errorCategory ?? '', ).toLowerCase(); return category === 'billing' && payload.retryable === true; } function isHardBillingFailurePayload( payload: Record | null, ): payload is Record { if (!payload) return false; // Billing-plane outages (503 BILLING_UNAVAILABLE) are transient infra, not // hard denials. Never let them reach the terminal, run-fatal hard-billing // path — they must stay repairable so the next run retries. if (isTransientBillingFailurePayload(payload)) return false; const category = String( payload.error_category ?? payload.errorCategory ?? '', ).toLowerCase(); const code = String(payload.code ?? payload.error_code ?? '').toUpperCase(); const message = String( payload.error ?? payload.message ?? payload.failure_description ?? '', ).toLowerCase(); if (category === 'billing') return true; if ( code === 'INSUFFICIENT_CREDITS' || code === 'BILLING_CAP_EXCEEDED' || code === 'MONTHLY_BILLING_LIMIT_EXCEEDED' ) { return true; } return ( (message.includes('billing cap') || message.includes('monthly billing limit') || message.includes('rolling 30-day organization billing cap') || message.includes('insufficient credits')) && !message.includes('rate limit') ); } /** * A normalized provider 402 is fatal only when the integration boundary has * declared it account-level capacity. A raw 402 never reaches this function as * proof by itself: providers use that status inconsistently. */ function isProviderAccountCapacityFailurePayload( payload: Record | null, ): payload is Record { if (!payload) return false; const code = String(payload.code ?? payload.error_code ?? '').toUpperCase(); const category = String( payload.error_category ?? payload.errorCategory ?? '', ).toLowerCase(); const origin = String( payload.failure_origin ?? payload.failureOrigin ?? '', ).toLowerCase(); return ( code === 'PROVIDER_ACCOUNT_CAPACITY' && category === 'provider_account' && (origin === 'provider' || origin === 'provider_account') ); } function normalizeHardBillingPayload( payload: Record, ): Record { return { kind: 'billing_cap_exceeded', code: getStringField(payload, 'code') ?? 'MONTHLY_BILLING_LIMIT_EXCEEDED', error_category: 'billing', failure_origin: getStringField(payload, 'failure_origin') ?? 'deepline_billing', message: getStringField(payload, 'error') ?? getStringField(payload, 'message') ?? 'Deepline billing cap exceeded.', ...payload, }; } function receiptFailureKindForToolErrorPayload( payload: Record | null, ): WorkReceiptFailureKind { // A transient billing-plane outage (503 BILLING_UNAVAILABLE) charged nothing // and is explicitly retryable. It must be repairable so the next run // re-executes instead of replaying the cached billing-infra failure. if (isTransientBillingFailurePayload(payload)) return 'repairable'; const code = getStringField(payload, 'code')?.toUpperCase(); // A credential connection can be added or replaced between play runs. Keep // the failure terminal within its owning run, but let a later run reclaim // the durable receipt instead of replaying stale authorization state. return code === 'INTEGRATION_CREDENTIALS_MISSING' ? 'repairable' : 'terminal'; } function formatHardBillingFailureMessage(input: { billing: Record; toolId: string; status: number; attempt: number; maxAttempts: number; }): string { const code = getStringField(input.billing, 'code'); const providerCapacity = isProviderAccountCapacityFailurePayload( input.billing, ); const message = getStringField(input.billing, 'message') ?? getStringField(input.billing, 'error') ?? (providerCapacity ? 'Provider account capacity blocked execution.' : 'Deepline billing cap exceeded.'); const headline = providerCapacity ? 'Provider account capacity blocked execution.' : 'Deepline billing cap exceeded.'; return `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${headline} Run halted before marking remaining rows processed. ${code ? `code=${code}. ` : ''}${message}`; } function formatInsufficientCreditsMessage(input: { billing: Record; toolId: string; }): string { const operation = getStringField(input.billing, 'operation_id') ?? getStringField(input.billing, 'operation') ?? input.toolId; const balance = formatCreditAmount(input.billing.balance_credits); const required = formatCreditAmount(input.billing.required_credits); const recommended = formatCreditAmount( input.billing.recommended_add_credits ?? input.billing.needed_credits, ); const billingUrl = getStringField(input.billing, 'billing_url'); const addSuffix = billingUrl && recommended !== '-' ? ` Add >=${recommended} at ${billingUrl}.` : billingUrl ? ` Add credits at ${billingUrl}.` : ''; return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`; } function formatPublicToolErrorPayload(input: { parsed: Record | null; bodyText: string; }): string { if (!input.parsed) { return input.bodyText.slice(0, 500); } const selected: Record = {}; for (const key of [ 'error', 'message', 'code', 'failure_origin', 'error_category', 'failure_description', 'operator_hint', 'failure_hint', 'details', 'provider', 'operation', 'request_id', 'requestId', 'credential_source', 'credential_owner', ]) { const value = input.parsed[key]; if (typeof value === 'string' && value.trim()) { selected[key] = value; } } return JSON.stringify( Object.keys(selected).length > 0 ? selected : input.parsed, ).slice(0, 1_500); } export function normalizeToolHttpErrorMessage(input: { toolId: string; status: number; attempt: number; maxAttempts: number; bodyText: string; retryable?: boolean; retryAfterMs?: number | null; schemaVersion?: ToolExecutionErrorSchemaVersion; }): ToolHttpError { const schemaVersion = input.schemaVersion ?? TOOL_EXECUTION_ERROR_SCHEMA_VERSION; let parsed: Record | null = null; try { const candidate = JSON.parse(input.bodyText); parsed = isRecord(candidate) ? candidate : null; } catch { parsed = null; } const serializedFailure = parsed?.tool_error; const hydratedFailure = deserializeToolExecutionFailure( '', serializedFailure, schemaVersion, ); const provider = getStringField(parsed, 'provider') ?? hydratedFailure?.provider ?? input.toolId.split(/[._:./-]+/)[0]?.trim() ?? null; const operation = getStringField(parsed, 'operation') ?? hydratedFailure?.operation ?? input.toolId; const code = getStringField(parsed, 'code') ?? hydratedFailure?.code ?? null; const rawOrigin = getStringField(parsed, 'failure_origin') ?? hydratedFailure?.origin; const rawCategory = getStringField(parsed, 'error_category') ?? hydratedFailure?.category; const origin = normalizeToolExecutionOrigin(rawOrigin, { provider, operation, }); const category = normalizeToolExecutionCategory(rawCategory); const trustworthy = origin !== 'unknown' && category !== 'unknown'; const publicOptions: Partial = { toolId: input.toolId, provider, operation, code, origin, category, retryable: trustworthy ? (hydratedFailure?.retryable ?? getBooleanField(parsed, 'retryable') ?? input.retryable === true) : false, statusCode: input.status, requestId: getStringField(parsed, 'request_id') ?? getStringField(parsed, 'requestId') ?? hydratedFailure?.requestId ?? null, retryAfterMs: hydratedFailure?.retryAfterMs ?? input.retryAfterMs ?? null, networkKind: hydratedFailure?.networkKind ?? (category === 'network' && code?.toUpperCase().includes('TIMEOUT') ? 'timeout' : null), networkScope: hydratedFailure?.networkScope ?? (origin === 'provider' && category === 'network' ? 'deepline_to_provider' : null), }; const billing = getObjectField(parsed, 'billing'); if (isInsufficientCreditsBilling(billing)) { return createToolHttpError( schemaVersion, `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage( { billing, toolId: input.toolId, }, )}`, billing, input.status, // Insufficient balance is fatal and non-retryable for the current run, // but a later explicit run must re-check the mutable workspace balance. 'repairable', { ...publicOptions, origin: 'deepline', category: 'billing', code: publicOptions.code ?? 'INSUFFICIENT_CREDITS', retryable: false, }, ); } const hardBillingPayload = isHardBillingFailurePayload(billing) ? normalizeHardBillingPayload(billing) : isHardBillingFailurePayload(parsed) ? normalizeHardBillingPayload(parsed) : isProviderAccountCapacityFailurePayload(parsed) ? parsed : null; if (hardBillingPayload) { const providerCapacity = isProviderAccountCapacityFailurePayload(hardBillingPayload); return createToolHttpError( schemaVersion, formatHardBillingFailureMessage({ billing: hardBillingPayload, toolId: input.toolId, status: input.status, attempt: input.attempt, maxAttempts: input.maxAttempts, }), hardBillingPayload, input.status, 'terminal', { ...publicOptions, origin: providerCapacity ? 'provider' : 'deepline', // `provider_account` is the integration-boundary category. The // portable ToolExecutionError taxonomy canonically represents it as // provider-owned authentication; the code retains the precise // account-capacity reason. category: providerCapacity ? 'authentication' : 'billing', code: providerCapacity ? 'PROVIDER_ACCOUNT_CAPACITY' : publicOptions.code, retryable: false, }, ); } return createToolHttpError( schemaVersion, `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatPublicToolErrorPayload( { parsed, bodyText: input.bodyText, }, )}`, billing, input.status, receiptFailureKindForToolErrorPayload(parsed), publicOptions, ); } export function extractErrorBilling( error: unknown, ): Record | null { return error instanceof ToolHttpError ? error.billing : null; } export function isHardBillingToolHttpError(error: unknown): boolean { if ( error instanceof ToolHttpError && (isInsufficientCreditsBilling(error.billing) || isHardBillingFailurePayload(error.billing) || isProviderAccountCapacityFailurePayload(error.billing)) ) { return true; } return ( error instanceof ToolExecutionError && ((error.origin === 'deepline' && error.category === 'billing' && error.code !== 'BILLING_UNAVAILABLE') || (error.origin === 'provider' && error.category === 'authentication' && error.code === 'PROVIDER_ACCOUNT_CAPACITY')) ); } /** * A tool call that ultimately failed with HTTP 429 — provider or * Deepline-internal rate-limit pushback. While the local retry budget is * active it feeds provider pacing; after exhaustion it becomes a row-scoped * Map Row Outcome unless the payload is a hard Deepline billing failure. */ export function isRateLimitToolHttpError(error: unknown): boolean { return error instanceof ToolHttpError && error.status === 429; } export function getToolHttpErrorReceiptFailureKind( error: unknown, ): WorkReceiptFailureKind | null { return error instanceof ToolHttpError ? error.receiptFailureKind : null; }