import { ToolExecutionError } from '../tool-execution-error'; const CLOUDFLARE_DURABLE_OBJECT_RESET_RE = /Durable Object.*(?:code (?:was|has been) updated|storage caused object)/; const CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE = /Too many subrequests by single Worker invocation/i; const CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE = /The requested endpoint could not be found, or you don't have access to it\. Please check the provided ID and try again\./i; const VERCEL_RUNTIME_API_DEPLOYMENT_MISSING_RE = /runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i; const RUNTIME_LIMIT_EXCEEDED_RE = /\bRUNTIME_LIMIT_EXCEEDED\b/i; const SANDBOX_CAPACITY_EXCEEDED_RE = /\bSANDBOX_CAPACITY_EXCEEDED\b/i; const RUNTIME_RUNNER_LOST_RE = /\bRUNTIME_RUNNER_LOST\b/i; const RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_RE = /\bRUNTIME_SANDBOX_INSPECTION_UNAVAILABLE\b/i; const RUNTIME_SANDBOX_LOST_RE = /\bRUNTIME_SANDBOX_LOST\b/i; const RUNTIME_SANDBOX_START_FAILED_RE = /\bRUNTIME_SANDBOX_START_FAILED\b/i; const RUNTIME_SANDBOX_OOM_RE = /\bRUNTIME_SANDBOX_OOM\b|(?:javascript heap out of memory|fatal error:.*(?:heap|allocation).*memory)/i; const RUNTIME_SANDBOX_KILLED_RE = /\bRUNTIME_SANDBOX_KILLED\b/i; const OUTPUT_TOO_LARGE_RE = /\b(?:OUTPUT_TOO_LARGE|OutputTooLarge)\b/; export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE = 'Run interrupted by a platform deploy. Deepline retries this automatically when possible; if this error is still visible, re-run the same command.'; export const PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE = 'Worker subrequest limit hit. Deepline retries automatically; if still visible, re-run.'; export const PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE = 'Cloudflare workflow control was temporarily unavailable. Deepline retries automatically; if still visible, re-run.'; export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE = 'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.'; export const RUNTIME_SANDBOX_LOST_MESSAGE = 'The execution sandbox became unreachable before returning a terminal result. Re-run the same command; if this keeps happening, contact Deepline support with the run ID.'; export const RUNTIME_SANDBOX_OOM_MESSAGE = 'The execution sandbox ran out of memory. Completed work is durably recorded. Deepline does not automatically retry this run because the same resource profile is unlikely to succeed. Reduce the batch size or row payload before starting a new run.'; export const RUNTIME_SANDBOX_KILLED_MESSAGE = 'The execution sandbox was killed before it could return a result, and the kill reason is unavailable. Completed work is durably recorded. Retry the same command safely; Deepline reuses completed steps and does not repeat their provider calls.'; export const RUNTIME_LIMIT_EXCEEDED_MESSAGE = 'The play reached its 30 minute runtime limit and was stopped. Completed row state was preserved; run a smaller batch or continue from the persisted rows.'; export const SANDBOX_CAPACITY_EXCEEDED_MESSAGE = 'Your organization has reached its concurrency limit, so this play did not start. Try again after another play finishes, or contact Deepline to request a higher limit.'; export const RUNTIME_RUNNER_LOST_MESSAGE = 'The execution runner stopped reporting liveness before returning a terminal result. Completed work is preserved, but Deepline did not automatically replay the run because provider side effects may already exist. Re-run only when it is safe to do so.'; export const RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_MESSAGE = 'The play stopped after Deepline could not verify the execution sandbox state. Completed row state was preserved. Re-run from the persisted rows; if this keeps happening, contact Deepline support with the run ID.'; // The sandbox was accepted but the runner never reported liveness, so the play // body never began. Safe to retry without the side-effect caveat RUNNER_LOST // carries: nothing ran, so no provider call can already exist. export const RUNTIME_SANDBOX_START_FAILED_MESSAGE = 'The execution sandbox never finished starting, so this play never began running. Re-run the same command; if this keeps happening, contact Deepline support with the run ID.'; export const WORKSPACE_STORAGE_NOT_READY_CODE = 'WORKSPACE_STORAGE_NOT_READY'; // User-facing message. Contains NO SQL, role names, database names, or hosts by // construction, so it is safe past redactRuntimeInternalError (which only strips // postgres-URI passwords). The raw pg error goes in the Error `cause` for server // logs only, never into this message or the normalized public failure. export const WORKSPACE_STORAGE_NOT_READY_MESSAGE = 'Workspace storage for this organization is unavailable, so this operation could not start. ' + 'This blocked operation did not use Deepline credits. ' + 'Earlier completed operations in the same run may already have used credits. ' + 'Re-run the command. If it fails again, ask an organization admin to run `deepline db repair` (POST /api/v2/ingestion/repair), then re-run. ' + 'If this keeps happening, contact Deepline support with the run ID.'; export class WorkspaceStorageNotReadyError extends Error { readonly code = WORKSPACE_STORAGE_NOT_READY_CODE; constructor(options?: { cause?: unknown }) { // Prefix the code token so `String(error)` and any stored error string // carry it. The run doc persists only the message string, and the read // path re-normalizes that string (extractPublicRunErrors), so the token // is how `errorCode` survives the serialization boundary back to the CLI. super( `${WORKSPACE_STORAGE_NOT_READY_CODE}: ${WORKSPACE_STORAGE_NOT_READY_MESSAGE}`, ); this.name = 'WorkspaceStorageNotReadyError'; // Attach `cause` manually instead of via the 2-arg Error constructor: // ErrorOptions is ES2022, but the Convex tsconfig typechecks this file // transitively under `lib: ES2021`, where the 2-arg form is a type error. if (options?.cause !== undefined) { (this as { cause?: unknown }).cause = options.cause; } } } export function isWorkspaceStorageNotReadyFailure(error: unknown): boolean { if (error instanceof WorkspaceStorageNotReadyError) return true; if (!error) return false; if (typeof error === 'object') { const code = (error as { code?: unknown }).code; if (code === WORKSPACE_STORAGE_NOT_READY_CODE) return true; const nestedErrors = (error as { errors?: unknown }).errors; if ( Array.isArray(nestedErrors) && nestedErrors.some(isWorkspaceStorageNotReadyFailure) ) { return true; } } const message = error instanceof Error ? error.message : String(error); return /\bWORKSPACE_STORAGE_NOT_READY\b/.test(message); } export const PROVIDER_EXHAUSTED_CODE = 'PROVIDER_EXHAUSTED'; /** * Format an absolute epoch-ms retry deadline as a human UTC string for the * user-facing PROVIDER_EXHAUSTED message. Contains no provider internals. */ export function formatProviderRetryAtUtc(retryAtMs: number): string { const millis = Number(retryAtMs); if (!Number.isFinite(millis)) { return 'an unknown time'; } return new Date(millis).toUTCString(); } /** * Build the user-facing PROVIDER_EXHAUSTED message. Names only the provider and * the retry time — never Deepline provider spend, SQL, or tenant internals — so * it is safe to surface verbatim past redaction. */ export function providerExhaustedMessage(input: { provider: string; retryAtMs: number; }): string { return ( `${input.provider} is exhausted and asked us to retry after ${formatProviderRetryAtUtc(input.retryAtMs)}. ` + 'This call was skipped with no spend. ' + 'For higher provider throughput guarantees, talk to us about an enterprise plan.' ); } /** * Thrown by the DB-authoritative pacer (`app_runtime_postgres` rate-state * backend) when a provider's hold exceeds the tolerable wait * (PROVIDER_EXHAUSTED_MAX_WAIT_MS). The call is skipped BEFORE any tool * execution request, so nothing is dispatched and nothing is billed. Row-failure * isolation still applies: this is a row/step outcome, not a run-fatal error. */ export class ProviderExhaustedError extends Error { readonly code = PROVIDER_EXHAUSTED_CODE; readonly provider: string; /** ISO 8601 retry deadline (from the server Retry-After / cooldown). */ readonly retryAt: string; constructor(input: { provider: string; retryAtMs: number; cause?: unknown }) { // Prefix the code token so `String(error)` and any persisted error string // carry it across the serialization boundary back to the CLI, exactly like // WorkspaceStorageNotReadyError. super( `${PROVIDER_EXHAUSTED_CODE}: ${providerExhaustedMessage({ provider: input.provider, retryAtMs: input.retryAtMs, })}`, ); this.name = 'ProviderExhaustedError'; this.provider = input.provider; this.retryAt = Number.isFinite(Number(input.retryAtMs)) ? new Date(Number(input.retryAtMs)).toISOString() : new Date(0).toISOString(); if (input.cause !== undefined) { (this as { cause?: unknown }).cause = input.cause; } } } /** * Derive the human provider name for a PROVIDER_EXHAUSTED error from a pacing * bucketId. Bucket ids are `${orgId}:${provider}`; the provider itself is * `tool:${toolId}` for undeclared tools (see `defaultPacingForTool`). Strip the * org prefix, then unwrap a `tool:` prefix down to the toolId so the message * names something the user recognizes. */ export function providerNameFromBucketId(bucketId: string): string { const separator = bucketId.indexOf(':'); const provider = separator >= 0 ? bucketId.slice(separator + 1) : bucketId; return provider.startsWith('tool:') ? provider.slice('tool:'.length) : provider; } export type PlayRunFailureDetails = { code: string; phase: string; message: string; retryable: boolean | null; cause?: string; name?: string; stack?: string; causes?: string[]; }; function formatRuntimeLimitDuration(timeoutSeconds: number): string { if (timeoutSeconds % 3_600 === 0) { const hours = timeoutSeconds / 3_600; return `${hours} ${hours === 1 ? 'hour' : 'hours'}`; } if (timeoutSeconds % 60 === 0) { const minutes = timeoutSeconds / 60; return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}`; } return `${timeoutSeconds} ${timeoutSeconds === 1 ? 'second' : 'seconds'}`; } export function runtimeLimitExceededCause(timeoutSeconds: number): string { return `RUNTIME_LIMIT_EXCEEDED: Play exceeded its configured ${timeoutSeconds} second runtime limit and was stopped.`; } export function runtimeLimitExceededMessage(timeoutSeconds: number): string { if (timeoutSeconds === 30 * 60) return RUNTIME_LIMIT_EXCEEDED_MESSAGE; return `The play reached its configured runtime limit of ${formatRuntimeLimitDuration(timeoutSeconds)} and was stopped. Completed row state was preserved; run a smaller batch or continue from the persisted rows.`; } export function runtimeLimitExceededFailure( timeoutSeconds: number, cause = runtimeLimitExceededCause(timeoutSeconds), ): PlayRunFailureDetails { return { code: 'RUNTIME_LIMIT_EXCEEDED', phase: 'runtime', message: runtimeLimitExceededMessage(timeoutSeconds), retryable: false, cause: boundedFailureText(cause), }; } const PUBLIC_FAILURE_STACK_LINE_LIMIT = 12; const PUBLIC_FAILURE_TEXT_BYTE_LIMIT = 64 * 1024; const PUBLIC_FAILURE_CAUSE_LIMIT = 8; const PUBLIC_FAILURE_CAUSE_LENGTH_LIMIT = 1_000; function boundedFailureText(value: string): string { const encoder = new TextEncoder(); if (encoder.encode(value).byteLength <= PUBLIC_FAILURE_TEXT_BYTE_LIMIT) { return value; } let bytes = 0; let output = ''; for (const character of value) { const characterBytes = encoder.encode(character).byteLength; if (bytes + characterBytes > PUBLIC_FAILURE_TEXT_BYTE_LIMIT) break; output += character; bytes += characterBytes; } return `${output}\n[truncated]`; } function boundedFailureStack(error: Error): string | undefined { if (typeof error.stack !== 'string' || !error.stack.trim()) return undefined; return boundedFailureText( error.stack .split('\n') .slice(0, PUBLIC_FAILURE_STACK_LINE_LIMIT) .join('\n'), ); } function failureCauseTexts(error: Error): string[] { const queue: unknown[] = []; const seen = new Set([error]); const causes: string[] = []; const withCause = error as Error & { cause?: unknown }; if (withCause.cause !== undefined) queue.push(withCause.cause); if (error instanceof AggregateError) queue.push(...error.errors); while (queue.length > 0 && causes.length < PUBLIC_FAILURE_CAUSE_LIMIT) { const current = queue.shift(); if (current === undefined || current === null || seen.has(current)) { continue; } seen.add(current); const text = toErrorText(current).trim(); if (text && text !== error.message && !causes.includes(text)) { causes.push(text.slice(0, PUBLIC_FAILURE_CAUSE_LENGTH_LIMIT)); } if (current instanceof Error) { const nested = current as Error & { cause?: unknown }; if (nested.cause !== undefined) queue.push(nested.cause); if (current instanceof AggregateError) queue.push(...current.errors); } } return causes; } function toErrorText(error: unknown): string { if (error instanceof Error) { return error.message; } return String(error); } export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails { const rawCause = toErrorText(error); const cause = boundedFailureText(rawCause); if ( error instanceof ToolExecutionError && error.origin === 'deepline' && error.category === 'billing' && error.code !== 'BILLING_UNAVAILABLE' ) { return { code: error.code?.trim() || 'BILLING_DENIED', phase: 'billing', message: cause, retryable: false, cause, }; } if ( (error && typeof error === 'object' && (error as { code?: unknown }).code === 'OUTPUT_TOO_LARGE') || OUTPUT_TOO_LARGE_RE.test(rawCause) ) { return { code: 'OUTPUT_TOO_LARGE', phase: 'runtime', message: cause, retryable: false, cause, }; } if (RUNTIME_LIMIT_EXCEEDED_RE.test(rawCause)) { const configuredSeconds = Number( /configured\s+(\d+)\s+second runtime limit/i.exec(rawCause)?.[1], ); return Number.isSafeInteger(configuredSeconds) && configuredSeconds > 0 ? runtimeLimitExceededFailure(configuredSeconds, cause) : runtimeLimitExceededFailure(30 * 60, cause); } if (SANDBOX_CAPACITY_EXCEEDED_RE.test(rawCause)) { return { code: 'SANDBOX_CAPACITY_EXCEEDED', phase: 'admission', message: SANDBOX_CAPACITY_EXCEEDED_MESSAGE, retryable: true, cause, }; } if (RUNTIME_RUNNER_LOST_RE.test(rawCause)) { const stack = error instanceof Error ? boundedFailureStack(error) : null; const causes = error instanceof Error ? failureCauseTexts(error) : []; return { code: 'RUNTIME_RUNNER_LOST', phase: 'infrastructure', message: RUNTIME_RUNNER_LOST_MESSAGE, retryable: false, cause, ...(error instanceof Error ? { name: error.name || 'Error' } : {}), ...(stack ? { stack } : {}), ...(causes.length > 0 ? { causes } : {}), }; } // Before the generic fallthrough could claim it. Without this branch the // whole minted string — sandbox and command UUIDs, `runner_heartbeat_not_observed`, // the embedded `schedulerReadinessResponses` JSON, `exit_code_file_unavailable` // — became the run's public `message` verbatim, and every surface that prints // a failure printed the scheduler talking to itself. The detail is not // swallowed: it stays on `cause`, which is what the run's logs and the CLI's // `errors[]` carry. if (RUNTIME_SANDBOX_START_FAILED_RE.test(rawCause)) { const stack = error instanceof Error ? boundedFailureStack(error) : null; const causes = error instanceof Error ? failureCauseTexts(error) : []; return { code: 'RUNTIME_SANDBOX_START_FAILED', phase: 'infrastructure', message: RUNTIME_SANDBOX_START_FAILED_MESSAGE, retryable: true, cause, ...(error instanceof Error ? { name: error.name || 'Error' } : {}), ...(stack ? { stack } : {}), ...(causes.length > 0 ? { causes } : {}), }; } if (RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_RE.test(rawCause)) { return { code: 'RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE', phase: 'infrastructure', message: RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_MESSAGE, retryable: false, cause, }; } if (RUNTIME_SANDBOX_OOM_RE.test(rawCause)) { return { code: 'RUNTIME_SANDBOX_OOM', phase: 'infrastructure', message: RUNTIME_SANDBOX_OOM_MESSAGE, // A confirmed runtime OOM is poison work for the current resource // profile. Retrying it automatically spends capacity without changing // the cause; completed receipts remain available to a deliberate run // after the input or limits are changed. retryable: false, cause, }; } if (RUNTIME_SANDBOX_KILLED_RE.test(rawCause)) { return { code: 'RUNTIME_SANDBOX_KILLED', phase: 'infrastructure', message: RUNTIME_SANDBOX_KILLED_MESSAGE, retryable: true, cause, }; } if (RUNTIME_SANDBOX_LOST_RE.test(rawCause)) { const stack = error instanceof Error ? boundedFailureStack(error) : null; const causes = error instanceof Error ? failureCauseTexts(error) : []; return { code: 'RUNTIME_SANDBOX_LOST', phase: 'infrastructure', message: RUNTIME_SANDBOX_LOST_MESSAGE, retryable: true, cause, ...(error instanceof Error ? { name: error.name || 'Error' } : {}), ...(stack ? { stack } : {}), ...(causes.length > 0 ? { causes } : {}), }; } if (CLOUDFLARE_DURABLE_OBJECT_RESET_RE.test(rawCause)) { return { code: 'PLATFORM_DEPLOY_INTERRUPTED', phase: 'runtime', message: PLATFORM_DEPLOY_INTERRUPTED_MESSAGE, retryable: true, cause, }; } if (VERCEL_RUNTIME_API_DEPLOYMENT_MISSING_RE.test(rawCause)) { return { code: 'PLATFORM_DEPLOY_INTERRUPTED', phase: 'runtime', message: PLATFORM_DEPLOY_INTERRUPTED_MESSAGE, retryable: true, cause, }; } if (CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE.test(rawCause)) { return { code: 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED', phase: 'runtime', message: PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE, retryable: true, cause, }; } if (CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE.test(rawCause)) { return { code: 'PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED', phase: 'runtime', message: PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE, retryable: true, cause, }; } const playDepthBudgetMatch = rawCause.match( /Play execution playDepth budget exceeded \((\d+)\/(\d+)\)\./, ); if (playDepthBudgetMatch) { return { code: 'PLAY_CALL_DEPTH_EXCEEDED', phase: 'runtime', message: `Play-call depth exceeded (${playDepthBudgetMatch[1]}/${playDepthBudgetMatch[2]}).`, retryable: false, cause, }; } if ( error instanceof ProviderExhaustedError || /\bPROVIDER_EXHAUSTED\b/.test(rawCause) ) { // The human message (provider + retry time) already lives in `cause` after // the code token prefix; surface it verbatim minus the token so the run // terminal reads cleanly. Contains no provider spend or tenant internals by // construction (see providerExhaustedMessage). const stripped = cause.replace( new RegExp(`^${PROVIDER_EXHAUSTED_CODE}:\\s*`), '', ); return { code: PROVIDER_EXHAUSTED_CODE, phase: 'runtime', message: stripped, retryable: false, cause, }; } if ( isWorkspaceStorageNotReadyFailure(error) || /\bWORKSPACE_STORAGE_NOT_READY\b/.test(rawCause) ) { return { code: WORKSPACE_STORAGE_NOT_READY_CODE, phase: 'storage', // Deliberately return the clean constant, NOT `cause`: the raw string may // carry the code token prefix, and we never surface tenant DB internals. message: WORKSPACE_STORAGE_NOT_READY_MESSAGE, retryable: false, }; } const lowerCause = cause.toLowerCase(); if ( lowerCause.includes('neondberror') || lowerCause.includes('bind message supplies') || lowerCause.includes('prepared statement') ) { return { code: 'RUN_STORAGE_FAILED', phase: 'storage', message: INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE, retryable: true, }; } const diagnostics = error instanceof Error ? { name: error.name || 'Error', stack: boundedFailureStack(error), causes: failureCauseTexts(error), } : null; return { code: 'RUN_FAILED', phase: 'runtime', message: cause, retryable: null, ...(diagnostics ? { name: diagnostics.name, ...(diagnostics.stack ? { stack: diagnostics.stack } : {}), ...(diagnostics.causes.length > 0 ? { causes: diagnostics.causes } : {}), } : {}), }; }