/** * PlayContextImpl — the cloud execution engine. * * Batching model: * 1. ctx.dataset("table_key", rows).withColumn("field", resolver).run() * admits a bounded row worker pool. * 2. ctx.tools.execute() calls inside admitted field resolvers QUEUE requests. * 3. The drain loop runs queued provider batches while row workers wait. * 4. Provider results resolve rows, and the next pending rows are admitted. * * Runtime integration: * - checkpoint: recovered on retry (skip completed batches) * - onBatchComplete: called after each provider batch for scheduler checkpointing */ import { AsyncLocalStorage } from 'async_hooks'; import { createDeferredPlayDataset, deserializeLegacyPlayDataset, deserializePlayDatasetCell, isPlayDataset, isSerializedPlayDataset, isSerializedPlayDatasetCell, iteratePlayDatasetInputPages, materializePlayDatasetInput, resolveMaterializeLimitCap, serializePlayDatasetCell, } from '@shared_libs/plays/dataset'; import type { PlayDataset, PlayDatasetInput } from '@shared_libs/plays/dataset'; import { compileRequestsWithStrategy, executeChunkedRequests, } from './batch-runtime'; import { PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS, runtimeLeaseHeartbeatIntervalFromExpiry, } from './lease-policy'; import { createRuntimeReceiptHeartbeatSupervisor } from './receipt-heartbeat-supervisor'; import { shouldRouteFixtureProvider, shouldRouteFixtureToolId, validateFixtureBehavior, waitForFixtureResponseDelay, } from './fixture-behavior'; import { dispatchBoundedSettled } from './bounded-dispatch'; import type { PlayQueueHint } from './governor/rate-state-backend'; import type { MapRowOutcome } from './durability-store'; import { stringifyPostgresJson } from './postgres-json'; import { RUNTIME_ROW_OUTPUT_NOT_JSON_SERIALIZABLE } from './runtime-sheet-row-transition'; import { RuntimeSheetRowWriter, type RuntimeSheetRowWriterDiagnostics, } from './runtime-sheet-row-writer'; import type { WorkReceiptFailureKind } from './work-receipts'; import { completedMapRowOutcome, failedMapRowOutcome, mapRowOutcomeRuntimeFields, resolveMapRowOutcomeKey, } from './map-row-outcome'; import { PLAY_RUNTIME_API_COMPAT_PATH, PLAY_RUNTIME_OPERATION_ATTEMPT_HEADER, PLAY_RUNTIME_OPERATION_ID_HEADER, } from './runtime-api-paths'; import { isRetryableSecretResolutionStatus, secretResolutionRetryDecision, SECRET_RESOLUTION_MAX_ATTEMPTS, SECRET_RESOLUTION_ATTEMPT_TIMEOUT_MS, type SecretResolutionRetryDecision, } from './secret-resolution-retry-policy'; import { createRootRunExecutionScope, deriveChildRunExecutionScope, type RunExecutionScope, } from './run-execution-scope'; import { DEFAULT_RUNTIME_EXECUTION_CAPABILITIES } from './execution-capabilities'; import { vercelProtectionBypassHeader } from './vercel-protection'; import { RUNTIME_RELIABILITY_POLICY } from './runtime-reliability-policy'; export { RuntimeSheetRowsBlockedError, type RuntimeSheetBlockedRowDetail, } from './runtime-sheet-errors'; import { createDefaultGovernanceSnapshot, createPlayExecutionGovernor, defaultPacingForTool, type GovernanceSnapshot, type PacingResolver, type PlayExecutionGovernor, } from './governor/governor'; import { resolveExecutionPolicy } from './governor/policy'; import { createRuntimeResourceGovernor, type RuntimeResourceGovernor, } from './resource-governor'; import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing'; import { legacyRawFromToolResponseRawV2, normalizeToolResponseContract, providerMetaFromToolResponseRawV2, RAW_V2_TOOL_RESPONSE_CONTRACT, type ToolResponseContract, type ToolResponseView, } from './tool-response-contract'; import { ProviderExhaustedError } from './run-failure'; import { buildPlayContractCompatibility, normalizePlayContractCompatibility, } from '@shared_libs/plays/contracts'; import { isProviderUnavailable, serializeToolExecutionFailure, TOOL_EXECUTION_ERROR_SCHEMA_VERSION, TOOL_EXECUTION_ERROR_SCHEMA_HEADER, type ToolExecutionErrorSchemaVersion, type ToolExecutionFailureV1, } from '../tool-execution-error'; import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend'; import { pacingPolicyForTool } from './pacing'; import { cloneToolExecuteResultWithExecution, attachToolResultListDataset, createToolExecuteResult, parseToolExecuteResponse, isToolExecuteResult, isSerializedToolExecuteResult, serializeToolExecuteResult, deserializeToolExecuteResult, type ParsedToolExecuteResponse, type ToolExecuteResult, type ToolResultExecutionMetadata, type ToolResultMetadataInput, } from './tool-result'; import { markToolExecuteResultExecutionOutcome, toolExecutionMetadataForOutcome, toolExecutionOutcomeForDurableReceipt, type DurableReceiptRecoverySource, type ToolExecutionOutcome, } from './tool-execution-outcome'; import { TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS, classifyToolExecuteHttpFailure, createToolExecuteHttpFailureAttemptTracker, parseToolExecuteRetryAfterMs, parseToolExecuteAuthScopeChangedError, ToolExecuteAuthScopeChangedError, } from './tool-execute-retry-policy'; import { createToolHttpError } from './tool-http-errors'; import { describeTransportError, transportGatewayOriginForDiagnostic, } from './transport-error-diagnostics'; import { buildDurableCtxCallCacheKey, buildDurableRunPlayInvocationScope, buildDurableToolAggregateProviderIdempotencyKey, buildDurableToolAggregateReceiptKey, buildDurableToolCallAuthScopeDigest, buildDurableToolCallCacheKey, buildDurableToolProviderIdempotencyKey, buildDurableToolReceiptPrefix, resolveDurableCallCachePolicy, } from './durable-call-cache'; import { PLAY_AUTHORING_CONTRACT_EDITION, normalizePlayAuthoringCustomerDbStatement, validateOptionalPlayAuthoringField, validatePlayAuthoringField, type PlayAuthoringContractEdition, type PlaySecretAuth, type PlaySecretAwareRequestInit, type PlaySecretPromise, type PlaySqlQuery, type PlayAuthoringRunScope, type PlayAuthoringRuntimeContext, } from '../plays/authoring-contract'; import { DURABLE_RECEIPT_WAIT_DELAY_MS, DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS, RuntimeReceiptLeaseLostError, RuntimeReceiptWaitTimeoutError, executeWithDurableRuntimeReceipt, resolveRuntimeToolReceiptWaitMaxAttempts, runtimeReceiptFailureError, runtimeReceiptFailureKindForError, runtimeReceiptOutput as durableRuntimeReceiptOutput, waitForCompletedRuntimeReceipt, waitForCompletedRuntimeReceipts, type DurableReceiptExecutionStore, } from './durable-receipt-execution'; import { QUERY_RESULT_DATASET_PAGE_SIZE, isCustomerDbDatasetTool, isQueryResultDatasetReadRequest, isQueryResultDatasetTool, } from './query-result-dataset'; import { isAlwaysFreshIntegrationTool } from './tool-cache-policy'; import { isRowIsolationExemptError } from './row-isolation'; import { createRuntimePersistenceLatch, RuntimePersistenceCircuitOpenError, tripRuntimePersistenceLatch, type RuntimePersistenceLatch, } from './persistence-latch'; import { assertCustomerOutputObjectWithinLimit, assertRuntimeReceiptOutputWithinLimit, } from './output-size-limits'; import { createRuntimeDatasetId } from './dataset-id'; import { dedupeExplicitMapKeyRows } from './map-row-identity'; import { deriveToolRequestIdentity, derivePlayRowIdentity, derivePlayRowIdentityFromKey, MAP_KEY_NAMESPACE_MAX_LENGTH, normalizeTableNamespace, sha256Hex, stableStringify, } from '@shared_libs/plays/row-identity'; import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline'; import { DEEPLINE_CELL_META_FIELD, previousCellFromValue, type PreviousCell, } from './cell-staleness'; import { cloneCsvAliasedRow, stripCsvProjectedFields, stripCsvProjectionMetadata, toSerializableCsvAliasedRow, } from './csv-rename'; import { createRuntimeMapMaterializationTracker, createRuntimeMapRetainedRowsTracker, estimateRuntimeMapRowsMemory, resolveRuntimeMapRowAdmission, NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER, NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES, runtimeMapJsonByteLength, resolveRuntimeMapMemoryLimits, RuntimeMapMemoryLimitError, } from './map-memory-limits'; import { setSpanAttributes, withActiveSpan } from './tracing'; import { DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE } from './runtime-constraints'; import { PlayExecutionSuspendedError, PlayRowExecutionSuspendedError, isPlayExecutionSuspendedError, isPlayRowExecutionSuspendedError, type PlayExecutionSuspension, } from './suspension'; import { createSecretRedactionContext, type SecretRedactionContext, } from './secret-redaction'; import { buildPlayDocflowNodeErrorPreview, buildPlayDocflowNodeInputPreviewMap, type PlayDocflowNodeInputCapture, } from './docflow-node-io'; import type { DocflowObservationUpsert } from './docflow-observation'; import { assertNoSecretTaint, assertSecretAuthUsesTls, createBase64SecretValue, createBearerSecretAuth, createHeaderSecretAuth, createPlaintextSecretPromise, createSecretConcat, createSecretHandle, isSecretAuthInput, isPlaintextSecretPromise, isSecretHandle, secretAuthEntries, secretAuthHeaderMarkers, valueContainsSecret, type SecretAuth, type SecretAuthValue, type SecretAuthInput, type SecretAwareRequestInit, type SecretHandle, type SecretValue, } from './secret-capability'; import type { CsvOptions, RowState, RuntimeDatasetOptions, ToolCallRequest, ToolBatchResult, ContextOptions, PlayCallOptions, PlayCheckpoint, PlayStep, PlayStepRowResult, PlayRowUpdate, MapFieldDefinition, MapFieldResolver, ToolCallOptions, RuntimeStepOptions, FetchOptions, ResolvedPlayExecution, PlayFetchResponse, MapExecutionScope, MapExecutionFrame, PlayExecutionEvent, IntegrationEventWaitHandler, RuntimeStepReceipt, RuntimeStepProgram, RuntimeStepProgramStep, RuntimeConditionalStepResolver, PlayCellProducerAttempt, PlayCellReadRef, PlayCellDecision, PlayRowReadOrder, PlayRowMeta, PlaySheetCellProducer, } from './ctx-types'; import { MAX_CELL_READ_REFS, MAX_ROW_READ_CELLS, ROW_META_CELL_KEY, cellReadCut, hydrateCellReadRefs, } from './cell-provenance'; import { buildPlayNodeScope, playNodeScopeToWire, type PlayNodeScope, } from './play-node-scope'; import { StepProgramDatasetBuilder, type StepProgramDatasetColumnInput, type StepProgramDatasetOptions, } from './step-program-dataset-builder'; import { readRuntimeSheetDatasetRows } from './runtime-api'; import { ctxRunPlayInlineOnlyMessage, resolveChildExecutionStrategy, } from './child-execution-strategy'; type ResolvedPlayExecutor = ( ctx: PlayContextImpl, input: Record, ) => Promise; /** * SECURITY: AsyncLocalStorage is per async execution, not a cross-run cache. * * Keep only row-scoped metadata here. Do not expand this store with credentials, * provider responses, org-wide mutable caches, or anything that must not cross * workflow boundaries if worker scheduling/interleaving changes. */ const rowContext = new AsyncLocalStorage<{ rowId: number; fieldName?: string; tableNamespace?: string; rowKey?: string; mapScope?: MapExecutionScope; mapStallObserver?: RuntimeMapStallObserver; mapStallResolverToken?: number; /** * Per-resolver inline-child fan-width guard. This stays in the async row * scope, so it becomes collectible as soon as the resolver settles. */ inlineChildInvocationNamespaces?: Set; }>(); type InlineCompositionStore = { context: PlayContextImpl; executionScope: RunExecutionScope; governor: PlayExecutionGovernor; playName: string; staticPipeline: ContextOptions['staticPipeline']; /** Immutable error contract pinned by the child play artifact. */ toolErrorSchemaVersion: ToolExecutionErrorSchemaVersion; /** Immutable authoring semantics pinned by the child play artifact. */ authoringContractEdition: PlayAuthoringContractEdition; /** Execute response contract pinned by the child artifact. */ toolResponseContract: ToolResponseContract; /** Explicit response-transform revision used for durable receipt identity. */ toolResponseReceiptRevision?: string; }; const inlineCompositionContext = new AsyncLocalStorage(); const toolExecutionOverrides = new AsyncLocalStorage<{ force: boolean }>(); const PROGRESS_HEARTBEAT_INTERVAL_MS = 1_000; const PURE_JS_HEARTBEAT_ROW_INTERVAL = 250; /** * How often map-frame checkpoint state is materialized (array spread + frame * clone) and a map.progress event is emitted during row execution. Per-row * materialization is O(rows²) in total copies and floods event consumers; a * row/time throttle keeps progress fresh while staying O(rows). */ const MAP_FRAME_FLUSH_ROW_INTERVAL = 100; /** Executed-row sheet persistence chunking: rows AND bytes, whichever first. */ const MAP_PERSIST_CHUNK_ROWS = 2_000; const MAP_PERSIST_CHUNK_BYTES = 8 * 1024 * 1024; const MAP_ROW_WRITER_BUFFER_BYTES = 16 * 1024 * 1024; const MAP_ROW_WRITER_MAX_FLUSH_MS = 100; const MAP_FRAME_FLUSH_INTERVAL_MS = 250; const MAP_STALL_LOG_INTERVAL_MS = 60_000; // Tool scheduling lanes wait at most this long for same-lane row continuations. // The window is fixed from the first item and never resets on later arrivals. const TOOL_BATCH_COALESCE_WINDOW_MS = 5; const TOOL_SCALAR_COALESCE_WINDOW_MS = 5; const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000; // The receipt gateway fully buffers the integration response before sending // headers to the runner. Once those headers arrive, a long body stall is a // broken gateway-to-runner connection, not slow provider work. export const TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS = 30_000; const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000; const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000; // Every runtime-API tool fetch needs a client-side deadline. Without one, a // stalled execute request (e.g. a credential-less call whose gateway connection // hangs before the fast 403 can return) leaves the row "1 in flight" for minutes // until the outer run deadline, violating loud-failure. The integrations execute // route caps server work at maxDuration=900s, so this ceiling sits just above it: // a healthy server (including the fast INTEGRATION_CREDENTIALS_MISSING 403) always // responds first, and only a genuinely stuck socket trips the abort — surfacing as // a bounded, repairable transport failure that row-isolation settles per row // instead of hanging the whole run. const DEFAULT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000 + 30_000; const FETCH_TRANSPORT_MAX_ATTEMPTS = RUNTIME_RELIABILITY_POLICY.egress.fetchMaxAttempts; const FETCH_TRANSPORT_RETRY_DELAY_MS = 100; const CTX_FETCH_HEADERS_TIMEOUT_MS = RUNTIME_RELIABILITY_POLICY.egress.fetchHeadersTimeoutMs; const CTX_FETCH_BODY_TIMEOUT_MS = RUNTIME_RELIABILITY_POLICY.egress.fetchBodyTimeoutMs; const CTX_FETCH_TOTAL_TIMEOUT_MS = RUNTIME_RELIABILITY_POLICY.egress.fetchTotalTimeoutMs; // cloudflared returns this branded HTML when its connection to the local app // resets. It is a transport failure before the app can reach a provider, not a // provider 502; require both the branded page and our development tunnel host. const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN = /\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i; const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100; const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25; // A newly claimed receipt should normally use its scheduled heartbeat. When a // test/SEV policy deliberately gives it only a moment of lease life, however, // scheduler setup and the first provider-dispatch turn can consume that entire // window before the timer fires. Renew once before fan-out in that narrow case. // Production's normal multi-minute lease remains on the zero-extra-query path. const IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS = 100; // Per-row sanity cap on distinct inline child invocations. `maxPlayCallDepth` // (governor) already bounds nesting; this bounds fan-WIDTH from one active row // resolver. The dedupe set is row-local, never retained for the full run. const MAX_INLINE_CHILD_INVOCATIONS_PER_ROW = 512; // Bound the retained failure detail carried in parent aggregates so a fully // failing large map cannot grow the progress event without limit. const MAX_INLINE_CHILD_FAILURE_DETAIL = 25; class ToolExecuteResponseBodyTransportError extends Error { readonly cause: unknown; constructor(cause: unknown) { super( cause instanceof Error ? cause.message : 'Tool execute response body transport failed.', ); this.name = 'ToolExecuteResponseBodyTransportError'; this.cause = cause; } } class ToolExecuteResponseBodyTimeoutError extends Error { constructor(toolId: string) { super( `Tool ${toolId} response body was not delivered within ${TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS}ms after response headers.`, ); this.name = 'ToolExecuteResponseBodyTimeoutError'; } } export class CtxFetchTimeoutError extends Error { readonly code = 'CTX_FETCH_TIMEOUT'; constructor( readonly phase: 'headers' | 'body' | 'total', readonly timeoutMs: number, ) { super( `[CTX_FETCH_TIMEOUT phase=${phase} timeout_ms=${timeoutMs}] ` + `ctx.fetch exceeded its ${phase} deadline after ${timeoutMs}ms.`, ); this.name = 'CtxFetchTimeoutError'; } } type CtxFetchDeadline = { bodyMs: number; headersMs: number; startedAt: number; totalMs: number; }; function positiveTimeoutMs( value: number | undefined, fallback: number, ): number { return value === undefined || !Number.isFinite(value) || value <= 0 ? fallback : Math.max(1, Math.floor(value)); } function remainingCtxFetchMs(deadline: CtxFetchDeadline): number { return Math.max(0, deadline.totalMs - (Date.now() - deadline.startedAt)); } async function runCtxFetchPhase(input: { callerSignal?: AbortSignal | null; controller: AbortController; deadline: CtxFetchDeadline; phase: 'headers' | 'body'; run: () => Promise; }): Promise { const remainingMs = remainingCtxFetchMs(input.deadline); if (remainingMs <= 0) { throw new CtxFetchTimeoutError('total', input.deadline.totalMs); } if (input.callerSignal?.aborted) { throw ( input.callerSignal.reason ?? new DOMException('Aborted', 'AbortError') ); } const phaseLimitMs = input.phase === 'headers' ? input.deadline.headersMs : input.deadline.bodyMs; const timeoutMs = Math.min(phaseLimitMs, remainingMs); const timeoutPhase = timeoutMs >= remainingMs ? 'total' : input.phase; let timeout: ReturnType | null = null; let onCallerAbort: (() => void) | null = null; const phaseTimeout = new Promise((_resolve, reject) => { timeout = setTimeout(() => { const error = new CtxFetchTimeoutError( timeoutPhase, timeoutPhase === 'total' ? input.deadline.totalMs : phaseLimitMs, ); reject(error); input.controller.abort(error); }, timeoutMs); }); const callerAbort = input.callerSignal ? new Promise((_resolve, reject) => { onCallerAbort = () => { const reason = input.callerSignal?.reason ?? new DOMException('Aborted', 'AbortError'); reject(reason); input.controller.abort(reason); }; input.callerSignal!.addEventListener('abort', onCallerAbort, { once: true, }); }) : null; try { return await Promise.race([ input.run(), phaseTimeout, ...(callerAbort ? [callerAbort] : []), ]); } finally { if (timeout) clearTimeout(timeout); if (onCallerAbort) { input.callerSignal?.removeEventListener('abort', onCallerAbort); } } } async function sleepWithinCtxFetchDeadline(input: { callerSignal?: AbortSignal | null; deadline: CtxFetchDeadline; delayMs: number; }): Promise { const controller = new AbortController(); await runCtxFetchPhase({ callerSignal: input.callerSignal, controller, deadline: input.deadline, phase: 'headers', run: () => new Promise((resolve) => { setTimeout( resolve, Math.min(input.delayMs, remainingCtxFetchMs(input.deadline)), ); }), }); } async function readCtxFetchBody(input: { callerSignal?: AbortSignal | null; controller: AbortController; deadline: CtxFetchDeadline; response: Response; }): Promise { if (!input.response.body) return ''; const reader = input.response.body.getReader(); const decoder = new TextDecoder(); try { return await runCtxFetchPhase({ callerSignal: input.callerSignal, controller: input.controller, deadline: input.deadline, phase: 'body', run: async () => { let body = ''; for (;;) { const chunk = await reader.read(); if (chunk.done) return body + decoder.decode(); body += decoder.decode(chunk.value, { stream: true }); } }, }); } catch (error) { // Cancellation is socket hygiene, not part of the timeout contract. A // hostile/custom stream may never resolve its cancel hook; awaiting it // here would recreate the exact forever-pending ctx.fetch this deadline // is meant to eliminate. void reader.cancel(error).catch(() => {}); throw error; } finally { reader.releaseLock(); } } class RuntimeMapStallObserver { readonly #resolvers = new Map< number, { phase: 'resolver' | 'fetch'; phaseStartedAt: number; startedAt: number; } >(); readonly #terminalAdmissionStartedAt = new Map(); readonly #terminalCommitStartedAt = new Map(); readonly #getWriterDiagnostics: () => RuntimeSheetRowWriterDiagnostics | null; readonly #log: (line: string) => void; readonly #mapName: string; readonly #pageOffset: number; readonly #intervalMs: number; #timer: ReturnType | null = null; #lastTerminalCommitAt = Date.now(); #terminalCommitted = 0; #token = 0; constructor(input: { getWriterDiagnostics: () => RuntimeSheetRowWriterDiagnostics | null; log: (line: string) => void; mapName: string; pageOffset: number; intervalMs?: number; }) { this.#getWriterDiagnostics = input.getWriterDiagnostics; this.#log = input.log; this.#mapName = input.mapName; this.#pageOffset = input.pageOffset; this.#intervalMs = positiveTimeoutMs( input.intervalMs, MAP_STALL_LOG_INTERVAL_MS, ); } resolverStarted(): { finish: () => void; token: number } { const token = ++this.#token; const now = Date.now(); this.#resolvers.set(token, { phase: 'resolver', phaseStartedAt: now, startedAt: now, }); this.#schedule(); return { token, finish: () => { this.#resolvers.delete(token); this.#cancelIfIdle(); }, }; } fetchStarted(resolverToken: number | undefined): () => void { if (resolverToken === undefined) return () => {}; const resolver = this.#resolvers.get(resolverToken); if (!resolver) return () => {}; resolver.phase = 'fetch'; resolver.phaseStartedAt = Date.now(); return () => { const current = this.#resolvers.get(resolverToken); if (!current) return; current.phase = 'resolver'; current.phaseStartedAt = Date.now(); }; } terminalQueued(): { admitted: () => void; committed: () => void; failed: () => void; } { const token = ++this.#token; const now = Date.now(); this.#terminalAdmissionStartedAt.set(token, now); this.#schedule(); return { admitted: () => { this.#terminalAdmissionStartedAt.delete(token); this.#terminalCommitStartedAt.set(token, Date.now()); }, committed: () => { this.#terminalAdmissionStartedAt.delete(token); if (this.#terminalCommitStartedAt.delete(token)) { this.#terminalCommitted += 1; this.#lastTerminalCommitAt = Date.now(); } this.#cancelIfIdle(); }, failed: () => { this.#terminalAdmissionStartedAt.delete(token); this.#terminalCommitStartedAt.delete(token); this.#cancelIfIdle(); }, }; } stop(): void { if (this.#timer) clearTimeout(this.#timer); this.#timer = null; } #schedule(): void { if (this.#timer) return; this.#timer = setTimeout(() => { this.#timer = null; this.#emitIfStalled(); if (this.#hasUnresolved()) this.#schedule(); }, this.#intervalMs); if ( typeof this.#timer === 'object' && 'unref' in this.#timer && typeof this.#timer.unref === 'function' ) { this.#timer.unref(); } } #cancelIfIdle(): void { if (!this.#hasUnresolved()) this.stop(); } #hasUnresolved(): boolean { return ( this.#resolvers.size > 0 || this.#terminalAdmissionStartedAt.size > 0 || this.#terminalCommitStartedAt.size > 0 ); } #emitIfStalled(): void { const now = Date.now(); if (now - this.#lastTerminalCommitAt < this.#intervalMs) return; const writer = this.#getWriterDiagnostics(); const unresolved = this.#resolvers.size + this.#terminalAdmissionStartedAt.size + this.#terminalCommitStartedAt.size + (writer?.activeRows ?? 0) + (writer?.queuedTerminalRows ?? 0) + (writer?.blockedTerminalRows ?? 0); if (unresolved === 0) return; const oldest = (...sources: Array>): number | null => { let value: number | null = null; for (const source of sources) { for (const candidate of source) { value = value === null ? candidate : Math.min(value, candidate); } } return value; }; const oldestUnsettledAt = oldest( [...this.#resolvers.values()].map((resolver) => resolver.startedAt), this.#terminalAdmissionStartedAt.values(), this.#terminalCommitStartedAt.values(), writer?.activeStartedAt === null || writer?.activeStartedAt === undefined ? [] : [writer.activeStartedAt], ); const phaseCandidates: Array<{ at: number; phase: 'fetch' | 'resolver' | 'writer_admission' | 'writer_commit'; }> = [ ...[...this.#resolvers.values()].map((resolver) => ({ at: resolver.startedAt, phase: resolver.phase, })), ...[...this.#terminalAdmissionStartedAt.values()].map((at) => ({ at, phase: 'writer_admission' as const, })), ...[...this.#terminalCommitStartedAt.values()].map((at) => ({ at, phase: 'writer_commit' as const, })), ]; if ( writer?.activeKind === 'terminal' && writer.activeRows > 0 && writer.activeStartedAt !== null ) { phaseCandidates.push({ at: writer.activeStartedAt, phase: 'writer_commit', }); } const oldestPhase = phaseCandidates.reduce< (typeof phaseCandidates)[number] | null >( (current, candidate) => current === null || candidate.at < current.at ? candidate : current, null, )?.phase; const fetchResolvers = [...this.#resolvers.values()].filter( (resolver) => resolver.phase === 'fetch', ); this.#log( `[runtime.map_stall] ${JSON.stringify({ map: this.#mapName, page_offset: this.#pageOffset, resolver_active: this.#resolvers.size, fetch_active: fetchResolvers.length, oldest_fetch_ms: Math.max( 0, now - (oldest( fetchResolvers.map((resolver) => resolver.phaseStartedAt), ) ?? now), ), terminal_admission_waiting: this.#terminalAdmissionStartedAt.size, terminal_commit_waiting: this.#terminalCommitStartedAt.size, terminal_write_active: writer?.activeKind === 'terminal' ? writer.activeRows : 0, terminal_queued: (writer?.queuedTerminalRows ?? 0) + (writer?.blockedTerminalRows ?? 0), terminal_committed: this.#terminalCommitted, oldest_unsettled_ms: Math.max(0, now - (oldestUnsettledAt ?? now)), oldest_phase: oldestPhase ?? 'resolver', last_terminal_commit_ms: Math.max(0, now - this.#lastTerminalCommitAt), })}`, ); } } async function readToolExecuteResponseBody(input: { toolId: string; abortController: AbortController | null; read: () => Promise; }): Promise { let timeoutHandle: ReturnType | null = null; const timeout = new Promise((_resolve, reject) => { timeoutHandle = setTimeout(() => { const error = new ToolExecuteResponseBodyTimeoutError(input.toolId); input.abortController?.abort(error); reject(error); }, TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS); }); try { return await Promise.race([input.read(), timeout]); } finally { if (timeoutHandle) clearTimeout(timeoutHandle); } } class ToolExecuteInvalidJsonError extends Error { readonly cause: unknown; constructor(cause: unknown) { super('Tool execute response body was not valid JSON.'); this.name = 'ToolExecuteInvalidJsonError'; this.cause = cause; } } // A single batched runtime-step-receipt request (get/claim/complete/fail/seed) // carries at most this many keys. At map scale a run can hold 5k-10k receipt // keys; sending them all in one request made the server claim/complete them // one-per-key serially in a single HTTP call, which crossed the ~100s origin // cap and 524'd (ADR-0012 receipt fence at 5k/10k rows). Chunking bounds each // request's server-side work while keeping per-key results faithful — chunks // are dispatched in order and their aligned result arrays concatenated, so the // key→receipt mapping is identical to the unchunked path. export const RUNTIME_STEP_RECEIPT_REQUEST_CHUNK_SIZE = 1_000; export function chunkArrayForReceiptRequest(items: readonly T[]): T[][] { if (items.length <= RUNTIME_STEP_RECEIPT_REQUEST_CHUNK_SIZE) { return items.length === 0 ? [] : [[...items]]; } const chunks: T[][] = []; for ( let start = 0; start < items.length; start += RUNTIME_STEP_RECEIPT_REQUEST_CHUNK_SIZE ) { chunks.push( items.slice(start, start + RUNTIME_STEP_RECEIPT_REQUEST_CHUNK_SIZE), ); } return chunks; } type SafeFetchModule = typeof import('@shared_libs/security/safe-fetch'); let safeFetchModule: Promise | null = null; export async function waitForNodeRuntimeMapRowsVisible(input: { mapName: string; tableNamespace: string; runId?: string | null; expectedRows: number; updatedRows: number; readVisibleRowCount: () => Promise; sleep?: (ms: number) => Promise; log?: (line: string) => void; }): Promise { if (input.expectedRows <= 0) return 0; const sleep = input.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); let lastVisibleRows = -1; for ( let attempt = 1; attempt <= NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS; attempt += 1 ) { lastVisibleRows = await input.readVisibleRowCount(); if (lastVisibleRows >= input.expectedRows) { if (attempt > 1) { input.log?.( `Runtime sheet visibility barrier satisfied ctx.dataset("${input.mapName}") ` + `after ${attempt} read(s): visible ${lastVisibleRows}/${input.expectedRows}; ` + `wrote ${input.updatedRows}/${input.expectedRows}; run ${input.runId ?? 'unknown'}.`, ); } return lastVisibleRows; } if (attempt === 1) { input.log?.( `Runtime sheet visibility barrier waiting ctx.dataset("${input.mapName}"): ` + `visible ${lastVisibleRows}/${input.expectedRows}; ` + `wrote ${input.updatedRows}/${input.expectedRows}; run ${input.runId ?? 'unknown'}.`, ); } if (attempt < NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS) { await sleep(NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS); } } throw new Error( `Runtime sheet visibility mismatch for ctx.dataset("${input.mapName}"): ` + `expected ${input.expectedRows} terminal visible row(s), saw ${lastVisibleRows}; ` + `write reported ${input.updatedRows}; run ${input.runId ?? 'unknown'}.`, ); } /** * Persisted sheet fields to layer onto a fresh map input row during seeding, * excluding any column the current run's input already provides. Input columns * stay authoritative (a persisted row is matched only by the map key and may * belong to a different run that merely shares that key); runtime fields like * __deeplineRowKey and re-derivable output columns — never present on the raw * input row — are restored. */ export function persistedFieldsAbsentFromInputRow( inputRow: Record, persisted: Record, ): Record { const extra: Record = {}; for (const key of Object.keys(persisted)) { if (!Object.prototype.hasOwnProperty.call(inputRow, key)) { extra[key] = persisted[key]; } } return extra; } export async function reconcileNodeRuntimeMapResultsWithPersistedSheet(input: { mapName: string; tableNamespace: string; runId?: string | null; expectedRows: number; currentRows: Record[]; failedRowCount: number; readPersistedRows: (input: { limit: number; offset: number; rowMode: 'all'; }) => Promise[]>; log?: (line: string) => void; }): Promise[]> { if ( input.failedRowCount > 0 || input.expectedRows <= 0 || input.currentRows.length >= input.expectedRows ) { return input.currentRows; } const persistedRows: Record[] = []; let offset = 0; while (persistedRows.length < input.expectedRows) { const page = await input.readPersistedRows({ limit: Math.min(10_000, input.expectedRows - persistedRows.length), offset, rowMode: 'all', }); if (page.length === 0) break; persistedRows.push(...page); offset += page.length; } if (persistedRows.length < input.expectedRows) { throw new Error( `Runtime sheet finalization mismatch for ctx.dataset("${input.mapName}"): ` + `expected ${input.expectedRows} terminal persisted row(s), saw ${persistedRows.length}; ` + `in-memory Node map results reported ${input.currentRows.length}; run ${input.runId ?? 'unknown'}.`, ); } if (persistedRows.length > input.currentRows.length) { input.log?.( `Runtime sheet finalization reconciled ctx.dataset("${input.mapName}") ` + `from ${input.currentRows.length}/${input.expectedRows} in-memory Node row(s) ` + `to ${persistedRows.length} terminal persisted row(s).`, ); return persistedRows; } return input.currentRows; } export function resolveToolRuntimeTimeoutMs( toolId: string, requestedTimeoutMs?: number, authoringContractEdition: PlayAuthoringContractEdition = PLAY_AUTHORING_CONTRACT_EDITION, ): number | undefined { if (requestedTimeoutMs !== undefined) { if (authoringContractEdition >= 2) { validatePlayAuthoringField( 'ctx.tools.execute.timeoutMs', requestedTimeoutMs, ); return requestedTimeoutMs; } // Edition 1 rounded positive finite values and treated every other value // as omitted. Preserve that behavior for already-published artifacts. if (Number.isFinite(requestedTimeoutMs) && requestedTimeoutMs > 0) { return Math.max(1, Math.ceil(requestedTimeoutMs)); } } const normalized = toolId.trim().toLowerCase(); // Long-inference tools keep their explicit 15-minute budget. Every other tool // gets the default ceiling so no runtime-API fetch is ever unbounded — a // missing-credential or otherwise stuck request fails loudly and fast within // the deadline instead of hanging "1 in flight". return normalized === 'deeplineagent' || normalized === 'deeplineagent_deeplineagent' || normalized === 'ai_inference' || normalized === 'deeplineagent_ai_inference' || normalized === 'aiinference' ? DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS : DEFAULT_TOOL_RUNTIME_TIMEOUT_MS; } function isDeeplineDeveloperTunnelOrigin502(input: { url: string; status: number; bodyText: string; }): boolean { if ( input.status !== 502 || !DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN.test(input.bodyText) ) { return false; } try { const hostname = new URL(input.url).hostname.toLowerCase(); return ( hostname === 'deeplinedeveloper.com' || hostname.endsWith('.deeplinedeveloper.com') ); } catch { return false; } } function isUnmarkedExecutionGateway502(input: { url: string; status: number; responseHeaders: Headers; }): boolean { if ( input.status !== 502 || input.responseHeaders.has('x-deepline-request-id') ) { return false; } try { return new URL(input.url).pathname.endsWith('/execute-fenced-v1'); } catch { return false; } } function loadSafeFetch(): Promise { safeFetchModule ??= import('@shared_libs/security/safe-fetch'); return safeFetchModule; } function waitForSecretResolutionRetry(ms: number): Promise { if (ms <= 0) return Promise.resolve(); return new Promise((resolve) => setTimeout(resolve, ms)); } function cancelRuntimeResponseBody(response: Response): void { try { void response.body?.cancel().catch(() => {}); } catch { // Connection cleanup must never replace the resolver failure or retry. } } const NO_SECRET_RESOLUTION_RETRY: SecretResolutionRetryDecision = { retry: false, retryDelayMs: 0, retryAfterMs: null, retryAfterExceedsCap: false, }; function isUnsafeOutboundUrlError(error: unknown): boolean { return error instanceof Error && error.name === 'UnsafeOutboundUrlError'; } function publicToolResponseEnvelope(value: unknown): { status: string; raw: unknown; rawV2?: unknown; view?: ToolResponseView; meta?: Record; } | null { if (!value || typeof value !== 'object' || Array.isArray(value)) { return null; } const record = value as Record; if (typeof record.status !== 'string') return null; const toolResponse = record.toolResponse; if ( !toolResponse || typeof toolResponse !== 'object' || Array.isArray(toolResponse) ) { return null; } const response = toolResponse as Record; const rawV2 = Object.prototype.hasOwnProperty.call(response, 'rawV2') ? response.rawV2 : undefined; const view = response.view === 'data' || response.view === 'rawV2' ? response.view : undefined; if ( !Object.prototype.hasOwnProperty.call(response, 'raw') && rawV2 === undefined ) { return null; } const providerMeta = providerMetaFromToolResponseRawV2( rawV2, view ?? 'rawV2', ); const toolResponseMeta = response.meta && typeof response.meta === 'object' && !Array.isArray(response.meta) ? (response.meta as Record) : {}; const responseMeta = response.responseMeta && typeof response.responseMeta === 'object' && !Array.isArray(response.responseMeta) ? (response.responseMeta as Record) : {}; return { status: record.status, raw: Object.prototype.hasOwnProperty.call(response, 'raw') ? response.raw : legacyRawFromToolResponseRawV2(rawV2, view ?? 'rawV2', responseMeta), ...(rawV2 !== undefined ? { rawV2 } : {}), ...(view ? { view } : {}), ...(Object.keys({ ...toolResponseMeta, ...providerMeta, ...responseMeta }) .length > 0 ? { meta: { ...toolResponseMeta, ...providerMeta, ...responseMeta } } : {}), }; } /** * A batched provider request returns one envelope for several logical source * tool calls. Each source call persists only its own item-shaped canonical * response: retaining the aggregate envelope would make its legacy `raw` * projection change on receipt replay and duplicate every batch for every row. */ function publicToolResponseForBatchedItem( execution: ParsedToolExecuteResponse, result: unknown, forceRawV2: boolean, ): ParsedToolExecuteResponse['toolResponse'] | undefined { const providerMeta = execution.toolResponse?.meta; if ( !forceRawV2 && (!execution.toolResponse || !Object.prototype.hasOwnProperty.call(execution.toolResponse, 'rawV2')) ) { return undefined; } return { rawV2: { data: result }, view: 'data', ...(providerMeta ? { meta: providerMeta } : {}), }; } /** * Batch splitters predate raw-v2 and receive the same legacy value they got * from callToolAPI. Keep that input contract stable; raw-v2 is attached to the * completed logical call separately by publicToolResponseForBatchedItem. */ function legacyResultForBatchSplitter( execution: ParsedToolExecuteResponse, ): unknown { if (execution.toolResponse && 'raw' in execution.toolResponse) { return execution.toolResponse.raw; } return execution.result != null && typeof execution.result === 'object' && !Array.isArray(execution.result) && 'data' in execution.result ? execution.result.data : execution.result; } const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata'; const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract'; const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent'; function recordOrNull(value: unknown): Record | null { return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record) : null; } function rowsFromUnknown(value: unknown): Record[] { if (!Array.isArray(value)) return []; return value.map((row) => row && typeof row === 'object' && !Array.isArray(row) ? (row as Record) : { value: row }, ); } function finiteNonNegativeInteger(value: unknown): number | null { if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { return null; } return Math.floor(value); } function finitePositiveInteger(value: unknown): number | null { const integer = finiteNonNegativeInteger(value); return integer !== null && integer > 0 ? integer : null; } /** * Sources that answered without opening a provider request of their own: a * completed-receipt cache hit, a worker-replay checkpoint, and a follower that * attached to an in-flight owner. `cached` is false on the in-flight metadata * (nothing was cached at attach time), but no request was made either, so * classifying it as a live call disagreed with the `reused` marker the sheet * already writes and inflated `providerRequests` against `reusedCells`. */ const REUSED_TOOL_EXECUTION_SOURCES = new Set([ 'cache', 'checkpoint', 'in_flight', ]); /** * Classify a settled tool-call result for the cell producer trace (ADR 0018). * Unknown shapes settle as `completed`: the call returned, which is the fact * the trace records. */ function toolAttemptOutcomeForResult( value: unknown, ): Exclude { if (value == null) return 'no_result'; if (typeof value !== 'object' || Array.isArray(value)) return 'completed'; const record = value as Record; const execution = (record._metadata as Record | undefined) ?.execution as Record | undefined; const reused = execution?.cached === true || execution?.receiptRole === 'follower' || (typeof execution?.source === 'string' && REUSED_TOOL_EXECUTION_SOURCES.has(execution.source)); if (reused) return 'cached'; return record.status === 'no_result' ? 'no_result' : 'completed'; } /** * Explicit execution location for a queued row-scoped tool call (ADR 0018). * `toolIdOverride` lets a native batch report the batch operation it actually * sent while keeping the members' column/namespace. */ function playNodeScopeForToolCallRequest( request: ToolCallRequest, toolIdOverride?: string, ): PlayNodeScope | null { return buildPlayNodeScope({ toolId: toolIdOverride ?? request.toolId, callKey: request.contextKey, column: request.fieldName, tableNamespace: request.tableNamespace, }); } type ToolExecutionApiOptions = { timeoutMs?: number; durableCallReceiptKey?: string | null; executionAuthScopeDigest?: string | null; /** * Historical receipt identity used for the API's provider/billing * idempotency receipt. This can differ from the worker-owned durable cache * receipt while a response contract migrates. */ providerIdempotencyReceiptKey?: string | null; providerIdempotencyKey?: string | null; receiptLeaseExpiresAt?: string | null; beforeProviderCall?: () => Promise | void; heartbeatReceipt?: () => Promise | void; parkProviderCall?: () => Promise | void; /** * Transfer the logical tool permit to the caller so it can remain held until * the decoded response is durably settled. The release callback is * idempotent. Callers must release it in a finally block. */ retainToolSlot?: (release: () => void) => void; /** * Explicit graph-node attribution for this provider call (ADR 0018). Carried * to the execute route, which stamps it onto the usage event so per-node * credits are a query instead of a log-text inference. */ playNodeScope?: PlayNodeScope | null; customerDbDataset?: { limit: number; offset: number; pageSize: number; totalRows: number; }; }; const IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT = 25; /** * Per-cell cap on the durable producer trace (ADR 0018). A cascade has a handful * of legs; this only bounds pathological loops, keeping `_cell_meta` small. */ const MAX_CELL_PRODUCER_ATTEMPTS = 12; const BATCH_SIZE_LOG_SAMPLE_LIMIT = 10; const STEP_PROGRAM_MAP_DEFINITION = Symbol('deepline.stepProgramMapDefinition'); function shouldPersistMapCellField(fieldName: string): boolean { return !fieldName.startsWith('_') || fieldName === '_metadata'; } function runtimeSheetPatchFieldName(fieldName: string): string { return fieldName.includes('.') ? sqlSafePlayColumnName(fieldName) : fieldName; } function isPlainRecord(value: unknown): value is Record { if (!value || typeof value !== 'object' || Array.isArray(value)) { return false; } const prototype = Object.getPrototypeOf(value); return prototype === Object.prototype || prototype === null; } type PersistableMapRow = MapRowOutcome; function comparePersistableMapRowsByInputIndex( left: PersistableMapRow, right: PersistableMapRow, ): number { const leftIndex = typeof left.inputIndex === 'number' && Number.isFinite(left.inputIndex) ? left.inputIndex : Number.POSITIVE_INFINITY; const rightIndex = typeof right.inputIndex === 'number' && Number.isFinite(right.inputIndex) ? right.inputIndex : Number.POSITIVE_INFINITY; return leftIndex - rightIndex; } function persistableMapRowIdentity(row: PersistableMapRow): string | null { if (row.key) return `key:${row.key}`; return typeof row.inputIndex === 'number' && Number.isFinite(row.inputIndex) ? `index:${Math.floor(row.inputIndex)}` : null; } function persistableMapRowBytes(row: PersistableMapRow): number { return JSON.stringify(row).length; } function assertPersistableMapRowWithinCustomerOutputLimits(input: { mapName: string; row: PersistableMapRow; }): void { const rowLabel = input.row.key?.trim() || (typeof input.row.inputIndex === 'number' && Number.isFinite(input.row.inputIndex) ? `input:${Math.floor(input.row.inputIndex)}` : 'unknown'); assertCustomerOutputObjectWithinLimit({ value: input.row.data, path: `ctx.dataset("${input.mapName}").row("${rowLabel}")`, }); } function failedMapRowLogLabel(row: PersistableMapRow): string { const rowId = row.data?.row_id; if (typeof rowId === 'string' && rowId.trim()) return rowId.trim(); if (row.key?.trim()) return row.key.trim(); if (typeof row.inputIndex === 'number' && Number.isFinite(row.inputIndex)) { return `input:${Math.floor(row.inputIndex)}`; } return 'unknown'; } type FieldMapRunResult = { completedRows: PersistableMapRow[]; failedRows: PersistableMapRow[]; completedRowCount: number; failedRowCount: number; previewRows: Record[]; retainedRowsComplete: boolean; }; type RuntimeMapRowPersistence = { persistRows: (rows: PersistableMapRow[]) => { admitted: Promise; committed: Promise; }; isPersisted: (row: PersistableMapRow) => boolean; checkpoint: (updates: PlayRowUpdate[]) => Promise; flush: () => Promise; diagnostics: () => RuntimeSheetRowWriterDiagnostics; }; function createRuntimeMapRowPersistence( persistRows: (rows: PersistableMapRow[]) => Promise, options?: { persistCheckpoint?: (updates: PlayRowUpdate[]) => Promise; onFailure?: (error: unknown) => void; }, ): RuntimeMapRowPersistence { const persisted = new Set(); const queued = new Set(); const writer = new RuntimeSheetRowWriter({ terminalKey: (row) => { const identity = persistableMapRowIdentity(row); if (!identity) { throw new Error('Runtime Sheet terminal row is missing an identity.'); } return identity; }, checkpointKey: (update) => `key:${update.key}`, mergeCheckpointUpdates: (current, incoming) => ({ ...current, ...incoming, dataPatch: { ...(current.dataPatch ?? {}), ...(incoming.dataPatch ?? {}), }, cellMetaPatch: { ...(current.cellMetaPatch ?? {}), ...(incoming.cellMetaPatch ?? {}), }, }), estimateTerminalBytes: persistableMapRowBytes, estimateCheckpointBytes: (update) => JSON.stringify(update).length, maxBatchRows: MAP_PERSIST_CHUNK_ROWS, maxBatchBytes: MAP_PERSIST_CHUNK_BYTES, maxBufferedBytes: MAP_ROW_WRITER_BUFFER_BYTES, maxFlushMs: MAP_ROW_WRITER_MAX_FLUSH_MS, onFailure: options?.onFailure, writeBatch: async (batch) => { if (batch.kind === 'terminal') { await persistRows([...batch.rows]); return { committed: batch.rows.length }; } if (batch.updates.length > 0) { await options?.persistCheckpoint?.([...batch.updates]); } return { committed: batch.updates.length }; }, }); return { persistRows: (rows) => { const admissions: Promise[] = []; const commits: Promise[] = []; const clearQueuedState = (identity: string | null) => { if (identity) { queued.delete(identity); } }; for (const row of rows) { const identity = persistableMapRowIdentity(row); if (identity && (persisted.has(identity) || queued.has(identity))) { continue; } if (identity) queued.add(identity); const settlement = writer.settle(row); admissions.push( settlement.admitted.catch((error) => { clearQueuedState(identity); throw error; }), ); commits.push( settlement.committed.then( () => { if (identity) { persisted.add(identity); clearQueuedState(identity); } }, (error) => { clearQueuedState(identity); throw error; }, ), ); } return { admitted: Promise.all(admissions).then(() => {}), committed: Promise.all(commits).then(() => {}), }; }, isPersisted: (row) => { const identity = persistableMapRowIdentity(row); return identity ? persisted.has(identity) : false; }, checkpoint: async (updates) => { await writer.checkpoint(updates); }, flush: async () => { await writer.finish(); }, diagnostics: () => writer.diagnostics(), }; } class FailFastMapRowsError extends Error { readonly completedRows: PersistableMapRow[]; readonly failedRows: PersistableMapRow[]; readonly cause: unknown; constructor(input: { cause: unknown; completedRows: PersistableMapRow[]; failedRows: PersistableMapRow[]; }) { super( input.cause instanceof Error ? input.cause.message : String(input.cause), ); this.name = 'FailFastMapRowsError'; this.cause = input.cause; this.completedRows = input.completedRows; this.failedRows = input.failedRows; } } function stableDigest(value: string): string { return sha256Hex(value); } // Receipt keys contain canonicalized tool inputs. Runtime diagnostics must never // emit them directly; a short digest is enough to correlate gateway, runner, // and waiter observations for one failing run. function runtimeReceiptKeyDigest(key: string): string { return stableDigest(key).slice(0, 12); } function runtimeReceiptReadSummary(input: { requested: readonly string[]; receipts: readonly (RuntimeStepReceipt | null | undefined)[]; resolved: ReadonlyMap; }): string { const statuses = input.receipts.reduce>( (counts, receipt) => { const status = receipt?.status ?? 'missing'; counts[status] = (counts[status] ?? 0) + 1; return counts; }, {}, ); const positionalMismatches = input.receipts.reduce( (count, receipt, index) => receipt && receipt.key.trim() !== (input.requested[index] ?? '').trim() ? count + 1 : count, 0, ); const missingRequested = input.requested.filter( (key) => !input.resolved.has(key), ); return ( `requested=${input.requested.length} returned=${input.receipts.length} ` + `resolved=${input.resolved.size} positional_mismatches=${positionalMismatches} ` + `request_digests=${input.requested.map(runtimeReceiptKeyDigest).join(',')} ` + `returned_digests=${input.receipts .map((receipt) => receipt ? runtimeReceiptKeyDigest(receipt.key) : 'null', ) .join(',')} ` + `missing_digests=${missingRequested .map(runtimeReceiptKeyDigest) .join(',')} ` + `statuses=${Object.entries(statuses) .map(([status, count]) => `${status}:${count}`) .join(',')}` ); } const runtimeReceiptReadTraceEnabled = process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'; function runtimeReceiptReadTrace(input: { keys: readonly string[]; receipts: Array; byKey: ReadonlyMap; }): string | null { if (!runtimeReceiptReadTraceEnabled) return null; const digest = (value: string) => stableDigest(value).slice(0, 16); const statusCounts = input.receipts.reduce>( (counts, receipt) => { if (receipt) { counts[receipt.status] = (counts[receipt.status] ?? 0) + 1; } return counts; }, {}, ); const expiredInFlightCount = input.receipts.filter((receipt) => { if ( receipt?.status !== 'queued' && receipt?.status !== 'pending' && receipt?.status !== 'running' ) { return false; } const expiresAt = receipt.leaseExpiresAt ? Date.parse(receipt.leaseExpiresAt) : Number.NaN; return Number.isFinite(expiresAt) && expiresAt <= Date.now(); }).length; return ( '[runtime-receipt-context.read-trace] ' + JSON.stringify({ requestedCount: input.keys.length, returnedCount: input.receipts.filter(Boolean).length, mapSize: input.byKey.size, statusCounts, expiredInFlightCount, requestedOrderDigest: digest(input.keys.join('\u0000')), returnedOrderDigest: digest( input.receipts .map((receipt) => receipt ? `${digest(receipt.key)}:${receipt.status}` : '-', ) .join('\u0000'), ), mapOrderDigest: digest( [...input.byKey.entries()] .map(([key, receipt]) => `${digest(key)}:${receipt.status}`) .join('\u0000'), ), }) ); } type DurableCtxOperation = 'step' | 'tool' | 'fetch'; function durableCtxKey(input: { orgId?: string | null; playId: string; operation: DurableCtxOperation; id: string; semanticKey?: string | null; staleAfterSeconds?: number | null; cacheEpochMs?: number; }): string { if (input.operation === 'tool') { throw new Error('Tool calls use tool receipt keys.'); } return buildDurableCtxCallCacheKey({ orgId: input.orgId, playId: input.playId, kind: input.operation, id: input.id, semanticKey: input.semanticKey, staleAfterSeconds: input.staleAfterSeconds, cacheEpochMs: input.cacheEpochMs, }); } function resolvedPlayRevisionFingerprint(play: ResolvedPlayExecution): string { const artifact = play.artifact ?? null; return stableDigest( stableStringify({ playId: play.playId, codeFormat: play.codeFormat ?? artifact?.codeFormat ?? null, artifactHash: artifact?.artifactHash ?? null, graphHash: artifact?.graphHash ?? null, sourceHash: artifact?.sourceHash ?? null, sourceCodeHash: typeof play.sourceCode === 'string' ? stableDigest(play.sourceCode) : null, }), ); } function displayNameFromProducerId(id: string): string { const withoutNamespace = id.split('/').at(-1) ?? id; return withoutNamespace .split(/[-_.\s]+/) .filter(Boolean) .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) .join(' '); } function compactRowResultsPreview(rows: T[]): T[] { if (rows.length <= IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT) { return rows; } return rows.slice(0, IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT); } function isRuntimeStepProgram(value: unknown): value is RuntimeStepProgram { return ( !!value && typeof value === 'object' && (value as { kind?: unknown }).kind === 'steps' && Array.isArray((value as { steps?: unknown }).steps) ); } function isRuntimeConditionalStepResolver( value: unknown, ): value is RuntimeConditionalStepResolver { return ( !!value && typeof value === 'object' && (value as { kind?: unknown }).kind === 'conditional' && typeof (value as { when?: unknown }).when === 'function' && typeof (value as { run?: unknown }).run === 'function' ); } /** * Prebuilt source is also typechecked by the deployed SDK, which can lag the * runtime. A tagged `runIf` keeps that source compatible with older `steps()` * declarations while retaining the explicit per-program waterfall opt-in. */ function continuesOnProviderUnavailable(program: RuntimeStepProgram): boolean { return ( program.continueOnProviderUnavailable === true || program.steps.some( (step) => isRuntimeConditionalStepResolver(step.resolver) && (step.resolver.when as { __deeplineProviderWaterfall?: unknown }) .__deeplineProviderWaterfall === true, ) ); } const CTX_MAP_MIGRATION_MESSAGE = 'ctx.map(...) has been replaced by ctx.dataset(...). Use ctx.dataset("rows", rows).withColumn("field", resolver).run().'; const DATASET_STEP_MIGRATION_MESSAGE = 'Dataset .step(...) has been replaced by .withColumn(...). Use .withColumn("field", resolver).'; class RuntimeDatasetBuilder> { private readonly builder: StepProgramDatasetBuilder< RuntimeStepProgramStep, RuntimeStepProgramStep['resolver'], RuntimeDatasetOptions, Promise>> >; constructor( private readonly ctx: PlayContextImpl, private readonly key: string, private readonly items: PlayDatasetInput, ) { this.builder = new StepProgramDatasetBuilder( (program, options) => this.ctx.runStepProgramMap( this.key, this.items, program as RuntimeStepProgram, options, ), { emptyColumnName: 'ctx.dataset(...).withColumn(name, ...) requires a non-empty column name.', invalidColumnsProgram: 'ctx.dataset(...).withColumns(...) requires a steps() program.', legacyStep: DATASET_STEP_MIGRATION_MESSAGE, }, ); } withColumn( name: string, resolver: StepProgramDatasetColumnInput, options?: StepProgramDatasetOptions, ): this { this.builder.withColumn(name, resolver, options); return this; } withColumns(program: RuntimeStepProgram): this { this.builder.withColumns(program); return this; } step(): never { return this.builder.step(); } run( options?: RuntimeDatasetOptions, ): Promise>> { return this.builder.run(options); } } const WAITING_ROW = Symbol('deepline.waiting_row'); const FAILED_ROW = Symbol('deepline.failed_row'); const COMPLETED_ROW = Symbol('deepline.completed_row'); type MapRowExecutionResult = | typeof COMPLETED_ROW | typeof WAITING_ROW | typeof FAILED_ROW; function normalizeFetchHeaders( headers: RequestInit['headers'], ): Record { if (!headers) return {}; if (headers instanceof Headers) { return Object.fromEntries( [...headers.entries()].map(([key, value]) => [key.toLowerCase(), value]), ); } if (Array.isArray(headers)) { return Object.fromEntries( headers.map(([key, value]) => [key.toLowerCase(), value]), ); } return Object.fromEntries( Object.entries(headers).map(([key, value]) => [ key.toLowerCase(), String(value), ]), ); } function parseJsonOrNull(bodyText: string): unknown | null { if (!bodyText.trim()) return null; try { return JSON.parse(bodyText) as unknown; } catch { return null; } } function assertJsonSerializableStepOutput( stepId: string, output: unknown, ): void { try { JSON.stringify(output); } catch (error) { throw new Error( `ctx.step(${stepId}) returned a value that cannot be checkpointed as JSON: ${ error instanceof Error ? error.message : String(error) }`, ); } } function normalizeStepDescription( value: string | undefined, ): string | undefined { const trimmed = value?.trim(); return trimmed ? trimmed : undefined; } function emptyCheckpoint(): PlayCheckpoint { return { completedBatches: {}, completedToolBatches: {}, resolvedWaterfalls: {}, mapFrames: {}, }; } function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame { return { ...frame, completedRowKeys: [...frame.completedRowKeys], pendingRowKeys: [...frame.pendingRowKeys], }; } /** * Adapt this runtime's per-tool queue hints into the Play Execution Governor's * pacing resolver. The Governor owns per-(org, provider) pacing; this only maps * the existing queue-hint metadata into the Governor's `PacingRule` shape. */ function createPacingResolver( getToolQueueHints: ContextOptions['getToolQueueHints'], getToolProvider: ContextOptions['getToolProvider'], ): PacingResolver { return async (toolId: string) => { const builtin = pacingPolicyForTool(toolId, []); if (builtin) return builtin; const hints = getToolQueueHints ? await getToolQueueHints(toolId) : []; const declared = pacingPolicyForTool(toolId, hints); if (declared) return declared; const provider = (await getToolProvider?.(toolId))?.trim(); if (!provider) return null; return { ...defaultPacingForTool(toolId, resolveExecutionPolicy('cjs_node20')), provider, }; }; } type ScalarPlayAuthoringRuntimeContext = Pick< PlayAuthoringRuntimeContext, | 'tools' | 'customerDb' | 'run' | 'tool' | 'step' | 'fetch' | 'secrets' | 'runPlay' | 'log' | 'sleep' >; export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext { private rowStates = new Map(); private toolCallQueue: ToolCallRequest[] = []; private pendingRuntimeToolOwnershipAssertions: Array<{ targets: Array<{ receiptKey: string; leaseId: string }>; resolve: () => void; reject: (error: unknown) => void; }> = []; private runtimeToolOwnershipAssertionFlushScheduled = false; private runtimeToolReceiptCompletionsInFlight = new Map(); /** * Direct durable boundaries (including ctx.fetch) arrive independently from * concurrent map rows. Hold one microtask's worth so they can use the bulk * receipt gateway instead of serial remote claim requests. */ private pendingRuntimeReceiptClaims = new Map< string, { scheduled: boolean; requests: Array<{ key: string; reclaimRunning: boolean; forceRefresh: boolean; forceFailedRefresh: boolean; resolve: (receipt: RuntimeStepReceipt | null) => void; reject: (error: unknown) => void; }>; } >(); /** * Completed durable receipt payloads can expand many times beyond their wire * size during JSON parsing. Scheduling groups may overlap for provider * throughput, but only one group at a time may hydrate and deliver cached * results. Pending claims leave this section before provider execution, so * cache misses retain normal cross-group concurrency. */ private runtimeReceiptHydrationTurn: Promise = Promise.resolve(); /** * Fixed, non-resetting coalescing deadlines for ready scheduling lanes. A * deadline starts when the first request enters an empty batch bucket. New * arrivals never push it back, so sustained traffic cannot starve dispatch. */ private readonly toolDispatchQueuedAtByLane = new Map(); private readonly toolDispatcherWakeWaiters = new Set<() => void>(); private toolDispatcherFailure: unknown | null = null; private toolCallResolvers = new Map< string, { resolve: (value: unknown) => void; reject: (reason: unknown) => void } >(); #options: ContextOptions; private readonly executionScope: RunExecutionScope; private logBuffer: string[] = []; private fixtureProviderPacingBypassLogged = false; private fixtureProviderPacingEnforcementLogged = false; private checkpoint: PlayCheckpoint; private readonly durableCallCacheEpochMs: number; /** * Durable tool receipts are the replay/cache authority for the execution * paths the host supports. Keeping the same completed result in this * checkpoint retained every provider payload for the lifetime of a run and * copied it again while serializing the terminal. Paths without a matching * receipt API keep the legacy checkpoint cache so local/in-process replay * and bulk-only hosts' direct calls still work. */ private readonly durableMappedToolResultsBackedByReceipts: boolean; private readonly durableDirectToolResultsBackedByReceipts: boolean; private steps: PlayStep[] = []; private explicitMapInvocationKeys = new Set(); /** The map step currently being built — substeps go here instead of top-level. */ private activeDatasetStep: Extract | null = null; /** Last completed map step — for post-map recordStep calls (e.g. run_javascript). */ private lastDatasetStep: Extract | null = null; private pureMapExecutionActive = false; /** * Active per-map collector of cell meta patches keyed by row key. Fed by * emitScopedRowUpdate during a map so executed rows persist with their * completed/cached cell meta — cross-run reuse decisions read that meta. */ private activeMapCellMeta: Map> | null = null; /** * Latest partial row patch per key for the active map. Terminal rows are * removed after their Runtime Sheet settlement commits; any remainder is the * exact partial-cell checkpoint that must cross the durability barrier before * an integration-event suspension is published. */ private activeMapCheckpointUpdates: Map | null = null; private lastProgressHeartbeatAt = 0; private pendingRowEventBoundaries: Array<{ boundaryId: string; eventKey: string; timeoutMs: number; }> = []; private processedRowCount = 0; private sleepBoundaryIndex = 0; private nextDocflowInvocationSequence = 0; /** * Whether this run captures the docflow trace at all (ADR 0019), decided at * admission and handed down through the signed runtime authority. * * Read at every capture SITE rather than only inside the recorders, because * the cost being gated is mostly in the arguments: `cellReadRefsForRow`, * `effectiveCellReads` and `rowInputReadColumns` each walk the row, and a * recorder that returns early has already paid for them. Off, an ungated org * writes precisely the `_cell_meta` it wrote before this feature existed — * no `reads`, no `decide`, no `_row` order — so nothing downstream has to * know two shapes. */ private get docflowCaptureEnabled(): boolean { return this.#options.docflowEnabled === true; } /** * `ctx.customerDb.query` is syntactic sugar over the query_customer_db tool. * Keep generated tool ids distinct so separate imperative queries retain * their normal mutation semantics. */ private customerDbQueryIndex = 0; private readonly secretRedactor: SecretRedactionContext = createSecretRedactionContext(); private mapInvocationIndex = 0; private readonly stepCallIndexByKey = new Map(); private readonly toolCallIndexByKey = new Map(); /** * Parent-level inline-child aggregates. Maintained only by the single-writer * progress path (never per child) so concurrent fan-out cannot contend. See * `flushInlineChildAggregates` and ADR 0013. */ private inlineChildAggregates: { total: number; ok: number; failed: number; failures: Array<{ childPlayName: string; error: string }>; } = { total: 0, ok: 0, failed: 0, failures: [] }; /** * Runtime persistence-failure circuit breaker (postgres_fast parity with the * Workers runtime). The first persistence failure — a receipt-completion * failure after a successful tool call, a receipt-failure write failure, or a * map-row sheet flush failure — trips this latch; the dispatch loops then stop * dispatching NEW provider calls so a dead tenant DB cannot keep billing calls * that have nowhere durable to land. */ private readonly persistenceLatch: RuntimePersistenceLatch = createRuntimePersistenceLatch(); /** * The single source of every concurrency, budget, and pacing decision for this * run-attempt. Tool/play/row slots are blocking semaphores; budgets are charged * via `chargeBudget`; child plays fork lineage-global counters via `forkChild`. */ private readonly governor: PlayExecutionGovernor; private readonly resourceGovernor: RuntimeResourceGovernor; private readonly resolvedPlayExecutorCache = new Map< string, Promise >(); readonly tools = { execute: (request: { id: string; tool: string; input: Record; description?: string; force?: boolean; staleAfterSeconds?: number | null; timeoutMs?: number; receiptWaitMs?: number; }): Promise => { if (!request || typeof request !== 'object' || Array.isArray(request)) { throw new Error( 'ctx.tools.execute requires a request object: ctx.tools.execute({ id, tool, input, description }).', ); } validatePlayAuthoringField('ctx.tools.execute.id', request.id); validatePlayAuthoringField('ctx.tools.execute.tool', request.tool); validatePlayAuthoringField('ctx.tools.execute.input', request.input); if (request.description !== undefined) { validatePlayAuthoringField( 'ctx.tools.execute.description', request.description, ); } if (request.force !== undefined) { validatePlayAuthoringField('ctx.tools.execute.force', request.force); } assertNoSecretTaint(request.input, 'ctx.tools.execute input'); if ( request.timeoutMs !== undefined && this.currentAuthoringContractEdition >= 2 ) { validatePlayAuthoringField( 'ctx.tools.execute.timeoutMs', request.timeoutMs, ); } if ( request.receiptWaitMs !== undefined && this.currentAuthoringContractEdition >= 2 ) { validatePlayAuthoringField( 'ctx.tools.execute.receiptWaitMs', request.receiptWaitMs, ); } const force = request.force === true || toolExecutionOverrides.getStore()?.force === true; return this.executeTool( request.id.trim(), request.tool, request.input, request.description || force || request.staleAfterSeconds !== undefined || request.timeoutMs !== undefined || request.receiptWaitMs !== undefined ? { ...(request.description ? { description: request.description } : {}), ...(force ? { force: true } : {}), ...(request.staleAfterSeconds !== undefined ? { staleAfterSeconds: request.staleAfterSeconds } : {}), ...(request.timeoutMs !== undefined ? { timeoutMs: request.timeoutMs } : {}), ...(request.receiptWaitMs !== undefined ? { receiptWaitMs: request.receiptWaitMs } : {}), } : undefined, ) as Promise; }, }; async tool( key: string, toolId: string, input: Record, options?: { description?: string }, ): Promise> { validatePlayAuthoringField('ctx.tool.key', key); validatePlayAuthoringField('ctx.tool.tool', toolId); validatePlayAuthoringField('ctx.tool.input', input); if (options?.description !== undefined) { validatePlayAuthoringField( 'ctx.tool.options.description', options.description, ); } return (await this.tools.execute>({ id: key, tool: toolId, input, ...(options?.description ? { description: options.description } : {}), })) as ToolExecuteResult; } async __deeplineRunWithForcedTools(run: () => Promise): Promise { return await toolExecutionOverrides.run({ force: true }, run); } readonly customerDb = { query: async >( statement: PlaySqlQuery | string, options?: { maxRows?: number; timeoutMs?: number }, ): Promise => { const sql = normalizePlayAuthoringCustomerDbStatement(statement); if (options?.maxRows !== undefined) { validatePlayAuthoringField( 'ctx.customerDb.query.options.maxRows', options.maxRows, ); } if (options?.timeoutMs !== undefined) { validatePlayAuthoringField( 'ctx.customerDb.query.options.timeoutMs', options.timeoutMs, ); } const result = (await this.tools.execute({ id: `customer_db_query_${this.customerDbQueryIndex++}`, tool: 'query_customer_db', input: { sql, ...(options?.maxRows !== undefined ? { max_rows: options.maxRows } : {}), }, description: 'Customer DB query', // The direct executor never reused a prior SQL result. Preserve that // imperative behavior for reads and mutations alike. force: true, ...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}), })) as { toolOutput?: { raw?: unknown } }; const raw = result.toolOutput?.raw; if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { throw new Error( 'query_customer_db returned an invalid Customer DB result.', ); } const rows = (raw as { rows?: unknown }).rows; if (rows === undefined) return []; if (!Array.isArray(rows)) { throw new Error( 'query_customer_db returned an invalid Customer DB rows result.', ); } return [...rows] as TRow[]; }, }; readonly secrets = { get: (name: string): PlaySecretPromise => { if (typeof name !== 'string' || !name.trim()) { throw new Error('ctx.secrets.get(name) requires a non-empty name.'); } const handle = createSecretHandle(name.trim()); const value = this.currentAuthoringContractEdition >= 4 ? createPlaintextSecretPromise(handle.name, () => this.resolveSecretHandle(handle), ) : handle; return value as unknown as PlaySecretPromise; }, bearer: (secret: SecretAuthValue): PlaySecretAuth => { return createBearerSecretAuth(secret) as unknown as PlaySecretAuth; }, header: (header: string, secret: SecretAuthValue): PlaySecretAuth => { return createHeaderSecretAuth( header, secret, ) as unknown as PlaySecretAuth; }, // Only editions 1–3 call these runtime-only compatibility helpers. Edition // 4's public authoring contract intentionally exposes normal strings. concat: (...parts: readonly (string | SecretValue)[]): SecretValue => createSecretConcat(parts), base64: (value: SecretValue): SecretValue => createBase64SecretValue(value), }; constructor(options: ContextOptions) { const fixtureBehavior = validateFixtureBehavior(options.fixtureBehavior); if (fixtureBehavior.ok === false) { throw new Error(fixtureBehavior.error); } if ( fixtureBehavior.behavior !== null && options.integrationMode !== 'fixture' ) { throw new Error( 'fixtureBehavior is only valid when integrationMode is fixture.', ); } options = { ...options, fixtureBehavior: fixtureBehavior.behavior, }; this.#options = options; this.checkpoint = options.checkpoint ?? emptyCheckpoint(); const checkpointCacheEpochMs = this.checkpoint.durableCallCacheEpochMs; this.durableCallCacheEpochMs = typeof checkpointCacheEpochMs === 'number' && Number.isFinite(checkpointCacheEpochMs) && checkpointCacheEpochMs >= 0 ? checkpointCacheEpochMs : Date.now(); this.checkpoint.durableCallCacheEpochMs = this.durableCallCacheEpochMs; this.durableMappedToolResultsBackedByReceipts = Boolean( (options.claimRuntimeStepReceipt || options.claimRuntimeStepReceipts) && (options.getRuntimeStepReceipt || options.getRuntimeStepReceipts) && (options.completeRuntimeStepReceipt || options.completeRuntimeStepReceipts), ); this.durableDirectToolResultsBackedByReceipts = Boolean( options.claimRuntimeStepReceipt && options.getRuntimeStepReceipt && options.completeRuntimeStepReceipt, ); if (this.durableDirectToolResultsBackedByReceipts) { // A resumed durable runner may receive a legacy checkpoint containing // full tool results. Drop that redundant copy immediately and recover // through the receipt store below. Bulk-only receipt contexts retain the // checkpoint because direct tool calls cannot use the bulk claim path. this.checkpoint.completedToolBatches = {}; } // The governance play id keys durable ctx receipts (durableCtxKey), the // cycle guard, and per-parent child-call counters. It must be the STABLE // play name — not the per-run workflow id — so receipts written by one // run are recoverable by the next run of the same play. The workers // harness seeds `currentPlayId: req.playName` through its explicit // governance snapshot; this fallback gives the node runner the same // cross-run semantics. const rootPlayId = options.playName?.trim() || options.playId || options.workflowId || 'anonymous-play'; const rootRunId = options.runId ?? options.workflowId ?? 'anonymous-run'; this.executionScope = options.executionScope ?? createRootRunExecutionScope({ runId: rootRunId, playId: rootPlayId, attempt: typeof options.runAttempt === 'number' && Number.isFinite(options.runAttempt) ? Math.max(0, Math.floor(options.runAttempt)) : 0, receiptNamespace: options.runtimeReceiptScope ?? rootPlayId, authority: { orgId: options.orgId ?? 'org', workflowId: options.workflowId ?? rootRunId, integrationMode: options.integrationMode ?? null, capabilities: DEFAULT_RUNTIME_EXECUTION_CAPABILITIES, }, }); if (options.requireSharedRateState && !options.rateState) { throw new Error( 'Shared rate-state backend is required for this Node runtime substrate.', ); } this.governor = options.executionGovernor ?? createPlayExecutionGovernor({ adapter: 'cjs_node20', scope: { orgId: options.orgId ?? 'org', rootRunId: options.runId ?? options.workflowId ?? 'run', }, rateState: options.rateState ?? new InMemoryRateStateBackend(), budgetState: options.budgetState, resolvePacing: createPacingResolver( options.getToolQueueHints, options.getToolProvider, ), resolveRateScope: options.getToolRateScope, maxConcurrentExternalCalls: options.maxConcurrentExternalCalls, maxConcurrentRows: options.maxConcurrentRows, // A root run keeps depth 0 (its first child play is depth 1) but still // seeds its own play id into the ancestry/currentPlayId so the cycle guard // catches a child re-invoking the root. createDefaultGovernanceSnapshot // couples depth to rootPlayId, so the root snapshot is built explicitly. resume: options.governance ?? { ...createDefaultGovernanceSnapshot({ orgId: options.orgId ?? 'org', rootRunId, }), currentPlayId: rootPlayId, ancestryPlayIds: [rootPlayId], }, }); this.resourceGovernor = createRuntimeResourceGovernor({ executionGovernor: this.governor, }); } /** @internal Compiler-injected authored-flow execution marker. */ async __deeplineDocflowHit(nodeId: string): Promise { const normalizedNodeId = nodeId.trim(); if (!normalizedNodeId) { throw new Error('Docflow runtime node id must be non-empty.'); } // These calls are COMPILER-INJECTED, so bundle-time gating cannot reach // them: a prebuilt is compiled once, with its diagram, and then runs for // every org. Without this check an ungated customer running a prebuilt gets // `docflow.node.*` events on the wire and a `step docflow:*` sequence in // `plays run --watch` — the vocabulary of a feature they do not have. // The id is still validated above: a malformed injection is a compiler bug // and stays loud whether or not the run is capturing. if (!this.docflowCaptureEnabled) return; this.emitExecutionEvent({ type: 'docflow.node.hit', nodeId: `docflow:${normalizedNodeId}`, at: Date.now(), }); } /** @internal Compiler-injected bounded authored-node observation boundary. */ __deeplineObserveDocflowNode( nodeId: string, inputCaptures: readonly PlayDocflowNodeInputCapture[], outputPaths: readonly string[], execute: () => T | Promise, ): T | Promise { const normalizedNodeId = nodeId.trim(); if (!normalizedNodeId) { throw new Error('Docflow runtime node id must be non-empty.'); } // Ungated, this boundary is a pass-through: the wrapped work still runs and // still returns what it returned, but nothing is previewed, redacted, or // emitted. Returning before `buildPlayDocflowNodeInputPreviewMap` is the // point — previewing every bound node's inputs is the expensive half, and a // customer outside the rollout should not pay for a capture they cannot // read. Same reasoning as `__deeplineDocflowHit` above: injected at compile // time, so the run is the only place left to decide. if (!this.docflowCaptureEnabled) return execute(); const attempt = this.currentRunAttempt; const inputs = buildPlayDocflowNodeInputPreviewMap(inputCaptures, { redactor: this.secretRedactor, }); const invocationSeq = this.nextDocflowInvocationSequence++; const invocationId = `${attempt}:${invocationSeq}`; const observationNodeId = `docflow:${normalizedNodeId}`; this.emitExecutionEvent({ type: 'docflow.node.started', nodeId: observationNodeId, attempt, invocationId, inputs: inputs.values, ...(inputs.truncated ? { inputsTruncated: true } : {}), at: Date.now(), }); this.emitDocflowObservation({ runId: this.currentGovernance.currentRunId, attempt, nodeId: observationNodeId, invocationSeq, invocationId, status: 'started', inputs: inputs.values, inputsTruncated: inputs.truncated, at: Date.now(), }); const complete = (result: T): void => { const outputRoots = new Set( outputPaths .map((path) => path.trim()) .filter((path) => path && path !== '$output') .map((path) => path.split('.')[0]), ); const outputs = buildPlayDocflowNodeInputPreviewMap( outputPaths .map((path) => path.trim()) .filter(Boolean) .map((path) => ({ path, readRoot: () => result, properties: path === '$output' ? [] : outputRoots.size > 1 ? path.split('.') : path.split('.').slice(1), })), { redactor: this.secretRedactor }, ); this.emitExecutionEvent({ type: 'docflow.node.completed', nodeId: observationNodeId, attempt, invocationId, inputs: inputs.values, ...(inputs.truncated ? { inputsTruncated: true } : {}), outputs: outputs.values, ...(outputs.truncated ? { outputsTruncated: true } : {}), at: Date.now(), }); this.emitDocflowObservation({ runId: this.currentGovernance.currentRunId, attempt, nodeId: observationNodeId, invocationSeq, invocationId, status: 'completed', inputs: inputs.values, inputsTruncated: inputs.truncated, outputs: outputs.values, outputsTruncated: outputs.truncated, at: Date.now(), }); }; const fail = (error: unknown): void => { const errorPreview = buildPlayDocflowNodeErrorPreview( error, this.secretRedactor, ); this.emitExecutionEvent({ type: 'docflow.node.failed', nodeId: observationNodeId, attempt, invocationId, inputs: inputs.values, ...(inputs.truncated ? { inputsTruncated: true } : {}), error: errorPreview, at: Date.now(), }); this.emitDocflowObservation({ runId: this.currentGovernance.currentRunId, attempt, nodeId: observationNodeId, invocationSeq, invocationId, status: 'failed', inputs: inputs.values, inputsTruncated: inputs.truncated, error: errorPreview, at: Date.now(), }); }; try { const result = execute(); if (result instanceof Promise) { void result.then(complete, fail); return result; } complete(result); return result; } catch (error) { fail(error); throw error; } } private durableBoundaryId(localId: string): string { // Durable boundaries live in one checkpoint for the whole root execution. // Nested plays and concurrent child calls therefore need a stable run scope // in the key, otherwise two children can both produce e.g. "sleep-0-25" // and replay the wrong boundary or never observe completion. return `${this.currentGovernance.currentRunId}:${localId}`; } private get activeInlineComposition(): InlineCompositionStore | null { const active = inlineCompositionContext.getStore(); return active?.context === this ? active : null; } /** Enforce the scalar inline-child contract at dynamic API boundaries. */ private assertInlineChildContract( reason: 'dataset_child' | 'suspending_child', ): void { const composition = this.activeInlineComposition; if (!composition) return; throw new Error(ctxRunPlayInlineOnlyMessage(composition.playName, reason)); } private get currentExecutionScope(): RunExecutionScope { return this.activeInlineComposition?.executionScope ?? this.executionScope; } private get currentExecutionGovernor(): PlayExecutionGovernor { return this.activeInlineComposition?.governor ?? this.governor; } private get currentGovernance(): GovernanceSnapshot { return this.currentExecutionGovernor.snapshot(); } private get currentPlayName(): string | undefined { return this.activeInlineComposition?.playName ?? this.#options.playName; } private get currentStaticPipeline(): ContextOptions['staticPipeline'] { const active = this.activeInlineComposition; return active ? active.staticPipeline : this.#options.staticPipeline; } /** * Inline children share this context's queues, but not its artifact contract. * Each child tool call must preserve the error shape pinned when that child * was published, even when the parent was published under another schema. */ private get currentToolErrorSchemaVersion(): ToolExecutionErrorSchemaVersion { // Historical artifacts must not turn durable structured failures back into // strings. Every runtime tool call and receipt rehydration uses v1. return TOOL_EXECUTION_ERROR_SCHEMA_VERSION; } private get currentToolResponseContract(): ToolResponseContract { return normalizeToolResponseContract( this.activeInlineComposition?.toolResponseContract ?? this.#options.toolResponseContract, ); } /** * Only explicitly declared response transformations affect receipt reuse. * Missing preserves historical artifact/receipt identity, while the public * response header still normalizes it to the legacy V2 behavior above. */ private get currentToolResponseReceiptRevision(): string | undefined { if (this.activeInlineComposition) { return this.activeInlineComposition.toolResponseReceiptRevision; } return this.#options.toolResponseReceiptRevision; } private get currentAuthoringContractEdition(): PlayAuthoringContractEdition { return ( this.activeInlineComposition?.authoringContractEdition ?? this.#options.authoringContractEdition ?? PLAY_AUTHORING_CONTRACT_EDITION ); } private recordPlayCallStep(input: { playId: string; execution?: 'inline'; description?: string | null; nestedSteps?: PlayStep[]; }): void { const step = { type: 'play_call' as const, playId: input.playId, ...(input.execution ? { execution: input.execution } : {}), nestedSteps: input.nestedSteps ?? [], description: normalizeStepDescription(input.description ?? undefined), }; if (this.activeDatasetStep) { this.activeDatasetStep.substeps.push(step); } else { this.steps.push(step); } } private async vercelProtectionHeaders(): Promise> { return vercelProtectionBypassHeader( this.#options.vercelProtectionBypassToken, ); } private emitScopedRowUpdate( key: string | null, tableNamespace: string | null, update: Omit, ): void { assertNoSecretTaint(update, 'ctx.dataset row update'); const rowScope = rowContext.getStore()?.mapScope; if (rowScope && key) { this.emitExecutionEvent({ type: 'map.row.updated', mapInvocationId: rowScope.mapInvocationId, mapNodeId: rowScope.mapNodeId ?? null, logicalNamespace: rowScope.logicalNamespace, artifactTableNamespace: rowScope.artifactTableNamespace, rowKey: key, rowStatus: update.status, fieldName: rowContext.getStore()?.fieldName ?? null, stage: update.stage ?? null, provider: update.provider ?? null, at: Date.now(), }); } if (key && update.cellMetaPatch && this.activeMapCellMeta) { const existingMeta = this.activeMapCellMeta.get(key); this.activeMapCellMeta.set( key, existingMeta ? { ...existingMeta, ...update.cellMetaPatch } : { ...update.cellMetaPatch }, ); } if (key && this.activeMapCheckpointUpdates) { const checkpointKey = `${tableNamespace ?? ''}\u0000${key}`; const existing = this.activeMapCheckpointUpdates.get(checkpointKey); const next: PlayRowUpdate = { ...(existing ?? { key, rowId: update.rowId, tableNamespace, }), key, rowId: update.rowId, tableNamespace, ...(update.status !== undefined ? { status: update.status } : {}), ...(update.stage !== undefined ? { stage: update.stage } : {}), ...(update.provider !== undefined ? { provider: update.provider } : {}), ...(update.error !== undefined ? { error: update.error } : {}), dataPatch: { ...(existing?.dataPatch ?? {}), ...(update.dataPatch ?? {}), }, cellMetaPatch: { ...(existing?.cellMetaPatch ?? {}), ...(update.cellMetaPatch ?? {}), }, // Row-grain provenance is re-stated whole on every patch, so the later // record always supersedes: it can only have grown (ADR 0019). ...((update.rowMetaPatch ?? existing?.rowMetaPatch) ? { rowMetaPatch: { ...(existing?.rowMetaPatch ?? {}), ...(update.rowMetaPatch ?? {}), }, } : {}), }; this.activeMapCheckpointUpdates.set(checkpointKey, next); } if (!key || !this.#options.onRowUpdate) { return; } void this.#options.onRowUpdate({ ...update, key, tableNamespace, }); } private clearActiveMapCheckpointUpdate( key: string, tableNamespace: string, ): void { this.activeMapCheckpointUpdates?.delete(`${tableNamespace}\u0000${key}`); } private emitExecutionEvent(event: PlayExecutionEvent): void { if (!this.#options.onExecutionEvent) { return; } try { const pending = this.#options.onExecutionEvent(event); if (pending && typeof pending.then === 'function') { void pending.catch(() => { // Execution events are observability projections. Their transport // must never change authored play behavior or mask its result. }); } } catch { // Keep synchronous observer failures outside the authored execution. } } /** * Durable docflow observation write (ADR 0016 rule 2). The sink is present * only for instrumented plays and posts to the receipt gateway. It is * strictly fire-and-forget: a synchronous throw or a rejected promise is * swallowed (loudly logged by the sink) so a durable-write failure can never * fail or slow the authored play body. The live event emission above is the * authoritative low-latency path and is unaffected. */ private emitDocflowObservation(observation: DocflowObservationUpsert): void { if (!this.#options.onDocflowObservation) { return; } try { const pending = this.#options.onDocflowObservation(observation); if (pending && typeof pending.then === 'function') { void pending.catch(() => { // Swallowed here; the sink is responsible for loud logging. }); } } catch { // Never let an observation write escape into authored execution. } } private async resolveSecretAuth(auth: SecretAuthInput | undefined) { const headers: Record = {}; for (const entry of secretAuthEntries(auth)) { Object.assign(headers, await this.resolveSingleSecretAuth(entry)); } return headers; } private async resolveSingleSecretAuth(auth: SecretAuth) { if (!auth) return {}; const value = await this.resolveSecretAuthValue(auth.secret); if (auth.kind === 'bearer') { return { authorization: `Bearer ${value}` }; } return { [auth.header.toLowerCase()]: value }; } private async resolveSecretAuthValue( secret: SecretAuthValue, ): Promise { if (typeof secret === 'string') { this.secretRedactor.register(secret); return secret; } if (isPlaintextSecretPromise(secret)) { const value = await secret; this.secretRedactor.register(value); return value; } return this.resolveSecretValue(secret as SecretValue); } private async resolveSecretValue(secret: SecretValue): Promise { if (isSecretHandle(secret)) return this.resolveSecretHandle(secret); const value = secret.kind === 'concat' ? ( await Promise.all( secret.parts.map((part) => typeof part === 'string' ? part : this.resolveSecretValue(part), ), ) ).join('') : Buffer.from( await this.resolveSecretValue(secret.value), 'utf8', ).toString('base64'); this.secretRedactor.register(value); return value; } private async resolveSecretHandle(secret: SecretHandle): Promise { let value: string | null = null; if (this.#options.resolveSecret) { value = await this.#options.resolveSecret({ name: secret.name, playName: this.#options.playName, orgId: this.#options.orgId, workflowId: this.#options.workflowId, runId: this.#options.runId, executorToken: this.#options.executorToken, }); } else if ( this.#options.baseUrl && this.#options.executorToken && this.#options.workflowId && this.#options.runId ) { const url = `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`; const operationId = `ctx-secret-operation-${crypto.randomUUID()}`; const operationStartedAt = Date.now(); for ( let attempt = 1; attempt <= SECRET_RESOLUTION_MAX_ATTEMPTS; attempt += 1 ) { const requestId = `ctx-secret-${crypto.randomUUID()}`; const attemptStartedAt = Date.now(); let response: Response; try { response = await fetch(url, { method: 'POST', // Bound the control-plane call so a stalled connection aborts and is // caught below as a transport failure instead of hanging the tool // call "1 in flight" forever. signal: AbortSignal.timeout(SECRET_RESOLUTION_ATTEMPT_TIMEOUT_MS), headers: { Authorization: `Bearer ${this.#options.executorToken}`, 'Content-Type': 'application/json', 'x-deepline-request-id': requestId, [PLAY_RUNTIME_OPERATION_ID_HEADER]: operationId, [PLAY_RUNTIME_OPERATION_ATTEMPT_HEADER]: String(attempt), ...(await this.vercelProtectionHeaders()), }, body: JSON.stringify({ action: 'resolve_secret', name: secret.name, playName: this.#options.playName, }), }); } catch (error) { const diagnostic = describeTransportError(error); const retry = secretResolutionRetryDecision({ attempt, retryAfter: null, }); this.log( `[runtime.secret_resolution_failure] ${JSON.stringify({ stage: 'transport', secret_name: secret.name, gateway_origin: transportGatewayOriginForDiagnostic(url), operation_id: operationId, request_id: requestId, attempt, max_attempts: SECRET_RESOLUTION_MAX_ATTEMPTS, retrying: retry.retry, retry_delay_ms: retry.retryDelayMs, elapsed_ms: Date.now() - attemptStartedAt, operation_elapsed_ms: Date.now() - operationStartedAt, error: diagnostic, })}`, ); if (!retry.retry) { throw new Error( `Secret ${secret.name} resolution transport failed (request_id=${requestId}): ${diagnostic.message ?? 'unknown transport error'}`, ); } await waitForSecretResolutionRetry(retry.retryDelayMs); continue; } const retryAfter = response.headers.get('retry-after'); const retry = isRetryableSecretResolutionStatus(response.status) ? secretResolutionRetryDecision({ attempt, retryAfter }) : NO_SECRET_RESOLUTION_RETRY; const responseDiagnostic = { stage: response.ok ? 'invalid_response' : 'http_response', secret_name: secret.name, gateway_origin: transportGatewayOriginForDiagnostic(url), operation_id: operationId, request_id: requestId, attempt, max_attempts: SECRET_RESOLUTION_MAX_ATTEMPTS, retrying: retry.retry, retry_delay_ms: retry.retryDelayMs, elapsed_ms: Date.now() - attemptStartedAt, operation_elapsed_ms: Date.now() - operationStartedAt, http_status: response.status, status_text: response.statusText || null, content_type: response.headers.get('content-type'), server: response.headers.get('server'), vercel_request_id: response.headers.get('x-vercel-id') ?? response.headers.get('x-vercel-request-id'), response_request_id: response.headers.get('x-deepline-request-id'), error_code: response.headers.get('x-deepline-error-code'), retry_after: retryAfter, retry_after_ms: retry.retryAfterMs, retry_after_exceeds_cap: retry.retryAfterExceedsCap, }; if (!response.ok) { this.log( `[runtime.secret_resolution_failure] ${JSON.stringify(responseDiagnostic)}`, ); cancelRuntimeResponseBody(response); if (retry.retry) { await waitForSecretResolutionRetry(retry.retryDelayMs); continue; } throw new Error( `Secret ${secret.name} is not available to this run (resolution status=${response.status}, request_id=${requestId}).`, ); } let payload: unknown; try { payload = await response.json(); } catch (error) { this.log( `[runtime.secret_resolution_failure] ${JSON.stringify({ ...responseDiagnostic, response_kind: 'invalid_json', parse_error_name: error instanceof Error ? error.name : typeof error, })}`, ); throw new Error( `Secret ${secret.name} is not available to this run (invalid resolution response, request_id=${requestId}).`, ); } value = payload && typeof payload === 'object' && !Array.isArray(payload) && typeof (payload as { value?: unknown }).value === 'string' ? (payload as { value: string }).value : null; if (!value) { this.log( `[runtime.secret_resolution_failure] ${JSON.stringify({ ...responseDiagnostic, response_kind: payload === null ? 'null' : Array.isArray(payload) ? 'array' : typeof payload, })}`, ); throw new Error( `Secret ${secret.name} is not available to this run (missing resolution value, request_id=${requestId}).`, ); } if (attempt > 1) { this.log( `[runtime.secret_resolution_recovered] ${JSON.stringify({ secret_name: secret.name, gateway_origin: transportGatewayOriginForDiagnostic(url), operation_id: operationId, request_id: requestId, attempts: attempt, operation_elapsed_ms: Date.now() - operationStartedAt, })}`, ); } break; } } else { throw new Error( 'ctx.secrets auth is unavailable because no secret resolver was configured.', ); } if (!value) { throw new Error(`Secret ${secret.name} is not available to this run.`); } this.secretRedactor.register(value); return value; } private setMapFrame(frame: MapExecutionFrame): void { this.checkpoint.mapFrames = { ...(this.checkpoint.mapFrames ?? {}), [frame.mapInvocationId]: cloneMapFrame(frame), }; } private createMapExecutionScope(input: { logicalNamespace: string; artifactTableNamespace: string; mapNodeId?: string | null; explicitKey?: | ((row: Record, index: number) => string) | null; }): MapExecutionScope { const mapInvocationId = `${input.logicalNamespace}:${this.mapInvocationIndex}`; this.mapInvocationIndex += 1; const explicitKey = input.explicitKey ?? null; return { mapInvocationId, mapNodeId: input.mapNodeId ?? null, logicalNamespace: input.logicalNamespace, artifactTableNamespace: input.artifactTableNamespace, rowIdentity: (row, index) => explicitKey ? derivePlayRowIdentityFromKey( explicitKey(row, index ?? 0), input.artifactTableNamespace, ) : derivePlayRowIdentity( stripCsvProjectedFields(row), input.artifactTableNamespace, ), }; } private normalizeContextKey(key: string, operation: string): string { const normalizedKey = key.trim(); if (!normalizedKey) { throw new Error(`ctx.${operation} requires a non-empty key.`); } return normalizedKey; } private async getRuntimeStepReceipt( key: string, ): Promise { if (!this.#options.getRuntimeStepReceipt) { return null; } const receipt = await this.#options.getRuntimeStepReceipt({ key }); if (receipt && typeof receipt.key === 'string' && receipt.key.trim()) { return { ...receipt, key: receipt.key.trim(), runId: receipt.runId ?? null, }; } return receipt ? { key: key.trim(), status: 'queued', } : null; } private async claimRuntimeStepReceipt( key: string, runId: string, reclaimRunning = false, forceRefresh = false, forceFailedRefresh = false, ): Promise { if (this.#options.claimRuntimeStepReceipts) { return await this.enqueueRuntimeStepReceiptClaim({ key, reclaimRunning, forceRefresh, forceFailedRefresh, }); } if (!this.#options.claimRuntimeStepReceipt) { return null; } const claimed = await this.#options.claimRuntimeStepReceipt({ key, leaseId: `receipt-lease:${crypto.randomUUID()}`, runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, leaseAware: true, ...(reclaimRunning ? { reclaimRunning: true } : {}), ...(forceRefresh ? { forceRefresh: true } : {}), ...(forceFailedRefresh ? { forceFailedRefresh: true } : {}), }); if (!claimed || typeof claimed.key !== 'string' || !claimed.key.trim()) { return null; } return { ...claimed, key: claimed.key.trim(), runId: claimed.runId ?? null, }; } private async enqueueRuntimeStepReceiptClaim(input: { key: string; reclaimRunning: boolean; forceRefresh: boolean; forceFailedRefresh: boolean; }): Promise { const batchKey = [ input.reclaimRunning ? 'reclaim' : 'ordinary', input.forceRefresh ? 'force' : 'cached', input.forceFailedRefresh ? 'failed-force' : 'failed-cached', ].join(':'); let batch = this.pendingRuntimeReceiptClaims.get(batchKey); if (!batch) { batch = { scheduled: false, requests: [] }; this.pendingRuntimeReceiptClaims.set(batchKey, batch); } return await new Promise((resolve, reject) => { batch!.requests.push({ ...input, resolve, reject }); if (batch!.scheduled) return; batch!.scheduled = true; queueMicrotask( () => void this.flushRuntimeStepReceiptClaimBatch(batchKey), ); }); } private async flushRuntimeStepReceiptClaimBatch( batchKey: string, ): Promise { const batch = this.pendingRuntimeReceiptClaims.get(batchKey); if (!batch) return; this.pendingRuntimeReceiptClaims.delete(batchKey); const requests = batch.requests; const first = requests[0]; const claimReceipts = this.#options.claimRuntimeStepReceipts; if (!first || !claimReceipts) { for (const request of requests) request.resolve(null); return; } try { const receipts = await this.dispatchChunkedRuntimeReceiptRequest( requests, (chunk) => { const leaseId = `receipt-lease:${crypto.randomUUID()}`; return claimReceipts({ keys: chunk.map((request) => request.key), leaseIds: chunk.map(() => leaseId), runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, leaseAware: true, ...(first.reclaimRunning ? { reclaimRunning: true } : {}), ...(first.forceRefresh ? { forceRefresh: true } : {}), ...(first.forceFailedRefresh ? { forceFailedRefresh: true } : {}), }); }, ); for (let index = 0; index < requests.length; index += 1) { const request = requests[index]!; request.resolve( this.normalizeRuntimeStepReceipt(request.key, receipts[index]), ); } } catch (error) { for (const request of requests) request.reject(error); } } private async completeRuntimeStepReceipt( key: string, _runId: string, output: unknown | null, leaseId?: string | null, ): Promise { assertNoSecretTaint(output, 'ctx.step receipt output'); assertRuntimeReceiptOutputWithinLimit({ output, path: `ctx.step receipt "${key}"`, }); if (!this.#options.completeRuntimeStepReceipt) { return null; } const completed = await this.#options.completeRuntimeStepReceipt({ key, runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, ...(leaseId ? { leaseId } : {}), output, }); if ( !completed || typeof completed.key !== 'string' || !completed.key.trim() ) { return null; } return { ...completed, key: completed.key.trim(), runId: completed.runId ?? null, }; } private async releaseRuntimeStepReceipt( key: string, _runId: string, leaseId?: string | null, ): Promise { if (!this.#options.releaseRuntimeStepReceipt) { return null; } const released = await this.#options.releaseRuntimeStepReceipt({ key, runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, ...(leaseId ? { leaseId } : {}), }); if (!released || typeof released.key !== 'string' || !released.key.trim()) { return null; } return { ...released, key: released.key.trim(), runId: released.runId ?? null, }; } private async heartbeatRuntimeStepReceipt( key: string, _runId: string, leaseId: string, ): Promise { if (!this.#options.heartbeatRuntimeStepReceipts) { return null; } const [receipt] = await this.#options.heartbeatRuntimeStepReceipts({ runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, leaseId, keys: [key], }); if (!receipt || typeof receipt.key !== 'string' || !receipt.key.trim()) { return null; } return { ...receipt, key: receipt.key.trim(), runId: receipt.runId ?? null, }; } private async assertRuntimeToolReceiptOwnership( requests: ToolCallRequest[], ): Promise { const targets = this.runtimeToolReceiptOwnershipTargets(requests); if (targets.length === 0) return; return await new Promise((resolve, reject) => { this.pendingRuntimeToolOwnershipAssertions.push({ targets, resolve, reject, }); if (this.runtimeToolOwnershipAssertionFlushScheduled) return; this.runtimeToolOwnershipAssertionFlushScheduled = true; queueMicrotask(() => void this.flushRuntimeToolOwnershipAssertionBatch()); }); } private runtimeToolReceiptOwnershipTargets( requests: ToolCallRequest[], ): Array<{ receiptKey: string; leaseId: string }> { return requests.flatMap((request) => { const receiptKey = request.receiptKey?.trim() || null; if (!receiptKey) return []; const leaseId = request.receiptLeaseId?.trim() || null; if (!leaseId) return []; return [{ receiptKey, leaseId }]; }); } private async renewRuntimeToolReceiptOwnership( requests: ToolCallRequest[], ): Promise { const targets = this.runtimeToolReceiptOwnershipTargets(requests); if (targets.length === 0) return; if (!this.#options.heartbeatRuntimeStepReceipts) { return await this.assertRuntimeToolReceiptOwnership(requests); } const uniqueTargets = [ ...new Map(targets.map((target) => [target.receiptKey, target])).values(), ]; const receipts = await this.#options.heartbeatRuntimeStepReceipts({ runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, keys: uniqueTargets.map((target) => target.receiptKey), leaseIds: uniqueTargets.map((target) => target.leaseId), }); this.assertRuntimeToolReceiptHeartbeatResults(uniqueTargets, receipts); } private async flushRuntimeToolOwnershipAssertionBatch(): Promise { this.runtimeToolOwnershipAssertionFlushScheduled = false; const assertions = this.pendingRuntimeToolOwnershipAssertions.splice(0); if (assertions.length === 0) return; const targets = assertions.flatMap((assertion) => assertion.targets); if (this.#options.heartbeatRuntimeStepReceipts) { try { const uniqueTargets = [ ...new Map( targets.map((target) => [target.receiptKey, target]), ).values(), ]; const receipts = await this.#options.heartbeatRuntimeStepReceipts({ runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, keys: uniqueTargets.map((target) => target.receiptKey), leaseIds: uniqueTargets.map((target) => target.leaseId), }); const renewed = new Map(); for (let index = 0; index < uniqueTargets.length; index += 1) { renewed.set( uniqueTargets[index]!.receiptKey, receipts[index] ?? null, ); } for (const assertion of assertions) { const lost = assertion.targets.find((target) => { const receipt = renewed.get(target.receiptKey); return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId); }); if (lost) { assertion.reject( new RuntimeReceiptLeaseLostError({ receiptKey: lost.receiptKey, runId: this.currentReceiptOwnerRunId, leaseId: lost.leaseId, }), ); } else { assertion.resolve(); } } } catch (error) { for (const assertion of assertions) assertion.reject(error); } return; } if ( this.#options.getRuntimeStepReceipt || this.#options.getRuntimeStepReceipts ) { try { const latest = await this.getRuntimeStepReceipts( targets.map((target) => target.receiptKey), ); for (const assertion of assertions) { const lost = assertion.targets.find((target) => { const receipt = latest.get(target.receiptKey); return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId); }); if (lost) { assertion.reject( new RuntimeReceiptLeaseLostError({ receiptKey: lost.receiptKey, runId: this.currentReceiptOwnerRunId, leaseId: lost.leaseId, }), ); } else { assertion.resolve(); } } } catch (error) { for (const assertion of assertions) assertion.reject(error); } return; } const error = new RuntimeReceiptLeaseLostError({ receiptKey: targets[0]?.receiptKey ?? 'unknown', runId: this.currentReceiptOwnerRunId, leaseId: targets[0]?.leaseId ?? 'unknown', }); for (const assertion of assertions) assertion.reject(error); } private assertRuntimeToolReceiptHeartbeatResults( targets: Array<{ receiptKey: string; leaseId: string }>, receipts: Array, ): void { for (let index = 0; index < targets.length; index += 1) { const target = targets[index]!; const receipt = this.normalizeRuntimeStepReceipt( target.receiptKey, receipts[index], ); if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) { throw new RuntimeReceiptLeaseLostError({ receiptKey: target.receiptKey, runId: this.currentReceiptOwnerRunId, leaseId: target.leaseId, }); } } } private runtimeToolReceiptStillOwned( receipt: RuntimeStepReceipt | null | undefined, leaseId: string, ): boolean { if (!receipt) return false; if (receipt.status === 'completed' || receipt.status === 'skipped') { return true; } if (receipt.status !== 'running') return false; if (receipt.leaseId !== leaseId) return false; const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null; return !ownerRunId || ownerRunId === this.currentReceiptOwnerRunId; } private runtimeToolReceiptStillOwnedByCurrentAttempt( receipt: RuntimeStepReceipt | null | undefined, leaseId: string | null | undefined, ): boolean { if (!receipt || receipt.status !== 'running') return false; if (leaseId?.trim()) { if (receipt.leaseId !== leaseId.trim()) return false; } else if (receipt.leaseId != null) { return false; } const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null; if (ownerRunId !== this.currentReceiptOwnerRunId) return false; const ownerAttempt = typeof receipt.leaseOwnerAttempt === 'number' ? receipt.leaseOwnerAttempt : 0; return ownerAttempt === this.currentRunAttempt; } private isOwnedClaimedRuntimeReceipt( receipt: RuntimeStepReceipt | null | undefined, ): receipt is RuntimeStepReceipt { return ( Boolean(receipt) && receipt?.claimState !== 'existing' && (receipt?.status === 'queued' || receipt?.status === 'pending' || receipt?.status === 'running') ); } private async failRuntimeStepReceipt( key: string, _runId: string, error: string, leaseId?: string | null, failureKind?: WorkReceiptFailureKind | null, errorPayload?: ToolExecutionFailureV1 | null, ): Promise { if (!this.#options.failRuntimeStepReceipt) { return null; } const failed = await this.#options.failRuntimeStepReceipt({ key, runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, ...(leaseId ? { leaseId } : {}), ...(failureKind ? { failureKind } : {}), ...(errorPayload ? { errorPayload } : {}), error, }); if (!failed || typeof failed.key !== 'string' || !failed.key.trim()) { return null; } return { ...failed, key: failed.key.trim(), runId: failed.runId ?? null, }; } private normalizeRuntimeStepReceipt( key: string, receipt: RuntimeStepReceipt | null | undefined, ): RuntimeStepReceipt | null { if (!receipt) return null; if (typeof receipt.key === 'string' && receipt.key.trim()) { return { ...receipt, key: receipt.key.trim(), runId: receipt.runId ?? null, }; } return { ...receipt, key: key.trim(), runId: receipt.runId ?? null, }; } /** * Dispatch a batched runtime-step-receipt request in bounded chunks and * concatenate the aligned result arrays back into one array positioned * identically to `items`. Every batch adapter returns receipts index-aligned * to the keys/receipts it was given, and chunks are dispatched sequentially in * input order, so the concatenation preserves the exact per-key mapping the * callers rely on. Sequential dispatch also keeps at most one receipt request * in flight, bounding tenant-DB and origin load at map scale. See * RUNTIME_STEP_RECEIPT_REQUEST_CHUNK_SIZE. */ private async dispatchChunkedRuntimeReceiptRequest( items: readonly TItem[], dispatchChunk: ( chunk: TItem[], ) => | Promise> | Array, ): Promise> { const chunks = chunkArrayForReceiptRequest(items); if (chunks.length <= 1) { return chunks.length === 0 ? [] : dispatchChunk(chunks[0]!); } const results: Array = []; for (const chunk of chunks) { const chunkResults = await dispatchChunk(chunk); for (const result of chunkResults) { results.push(result); } } return results; } private async getRuntimeStepReceipts( keys: string[], ): Promise> { const uniqueKeys = [ ...new Set(keys.map((key) => key.trim()).filter(Boolean)), ]; if (uniqueKeys.length === 0) return new Map(); const getReceipts = this.#options.getRuntimeStepReceipts; const receipts = getReceipts ? await this.dispatchChunkedRuntimeReceiptRequest(uniqueKeys, (chunk) => getReceipts({ keys: chunk }), ) : await Promise.all( uniqueKeys.map((key) => this.getRuntimeStepReceipt(key)), ); const byKey = new Map(); for (let index = 0; index < receipts.length; index += 1) { const normalized = this.normalizeRuntimeStepReceipt( uniqueKeys[index] ?? '', receipts[index], ); if (normalized) byKey.set(normalized.key, normalized); } if (runtimeReceiptReadTraceEnabled) { this.log( `[runtime-receipt-normalize] ${runtimeReceiptReadSummary({ requested: uniqueKeys, receipts, resolved: byKey, })}`, ); const trace = runtimeReceiptReadTrace({ keys: uniqueKeys, receipts, byKey, }); if (trace) this.log(trace); } return byKey; } private async claimRuntimeStepReceipts( keys: string[], _runId: string, reclaimRunning = false, forceRefresh = false, forceFailedRefresh = false, ): Promise> { const uniqueKeys = [ ...new Set(keys.map((key) => key.trim()).filter(Boolean)), ]; if (uniqueKeys.length === 0) return new Map(); const claimReceipts = this.#options.claimRuntimeStepReceipts; const receipts = claimReceipts ? await this.dispatchChunkedRuntimeReceiptRequest(uniqueKeys, (chunk) => { const leaseId = `receipt-lease:${crypto.randomUUID()}`; return claimReceipts({ keys: chunk, // A transport retry can replay this mutation after the store // committed but before the response body was consumed. Stable // caller-owned tokens distinguish that replay from a concurrent // claimant without weakening the provider-call execution fence. leaseIds: chunk.map(() => leaseId), runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, leaseAware: true, ...(reclaimRunning ? { reclaimRunning: true } : {}), ...(forceRefresh ? { forceRefresh: true } : {}), ...(forceFailedRefresh ? { forceFailedRefresh: true } : {}), }); }) : await Promise.all( uniqueKeys.map((key) => this.claimRuntimeStepReceipt( key, this.currentReceiptOwnerRunId, reclaimRunning, forceRefresh, forceFailedRefresh, ), ), ); const byKey = new Map(); for (let index = 0; index < receipts.length; index += 1) { const normalized = this.normalizeRuntimeStepReceipt( uniqueKeys[index] ?? '', receipts[index], ); if (normalized) byKey.set(normalized.key, normalized); } if (runtimeReceiptReadTraceEnabled) { this.log( `[runtime-receipt-claim-normalize] reclaim_running=${reclaimRunning} force_refresh=${forceRefresh} force_failed_refresh=${forceFailedRefresh} ` + runtimeReceiptReadSummary({ requested: uniqueKeys, receipts, resolved: byKey, }), ); } return byKey; } private async withRuntimeReceiptHydrationTurn( hydrate: () => Promise, ): Promise { const precedingTurn = this.runtimeReceiptHydrationTurn; let releaseTurn!: () => void; this.runtimeReceiptHydrationTurn = new Promise((resolve) => { releaseTurn = resolve; }); await precedingTurn; try { return await hydrate(); } finally { releaseTurn(); } } private async markRuntimeStepReceiptRunning(input: { key: string; runId: string; leaseId?: string | null; }): Promise { if (!this.#options.markRuntimeStepReceiptRunning) { if (input.leaseId && this.#options.heartbeatRuntimeStepReceipts) { return await this.heartbeatRuntimeStepReceipt( input.key, input.runId, input.leaseId, ); } return await this.getRuntimeStepReceipt(input.key); } const receipt = await this.#options.markRuntimeStepReceiptRunning({ key: input.key, runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, ...(input.leaseId ? { leaseId: input.leaseId } : {}), }); return this.normalizeRuntimeStepReceipt(input.key, receipt); } private async markRuntimeStepReceiptsRunning( receipts: Array<{ key: string; runId: string; leaseId?: string | null; }>, ): Promise> { const normalizedInputs = receipts.filter((receipt) => receipt.key.trim()); if (normalizedInputs.length === 0) return new Map(); let marked: Array; const markReceipts = this.#options.markRuntimeStepReceiptsRunning; if (markReceipts) { marked = await this.dispatchChunkedRuntimeReceiptRequest( normalizedInputs.map((receipt) => ({ ...receipt, runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, })), (chunk) => markReceipts({ receipts: chunk }), ); } else { marked = await Promise.all( normalizedInputs.map((receipt) => this.markRuntimeStepReceiptRunning(receipt), ), ); } const byKey = new Map(); for (let index = 0; index < marked.length; index += 1) { const normalized = this.normalizeRuntimeStepReceipt( normalizedInputs[index]?.key ?? '', marked[index], ); if (normalized) byKey.set(normalized.key, normalized); } return byKey; } private async markRuntimeStepReceiptsQueued( receipts: Array<{ key: string; runId: string; leaseId?: string | null; }>, ): Promise> { const normalizedInputs = receipts.filter((receipt) => receipt.key.trim()); if (normalizedInputs.length === 0) return new Map(); if (!this.#options.markRuntimeStepReceiptsQueued) { return new Map(); } const queued = await this.dispatchChunkedRuntimeReceiptRequest( normalizedInputs.map((receipt) => ({ ...receipt, runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, })), (chunk) => this.#options.markRuntimeStepReceiptsQueued!({ receipts: chunk }), ); const byKey = new Map(); for (let index = 0; index < queued.length; index += 1) { const normalized = this.normalizeRuntimeStepReceipt( normalizedInputs[index]?.key ?? '', queued[index], ); if (normalized) byKey.set(normalized.key, normalized); } return byKey; } private async parkRuntimeToolReceiptsQueued( requests: ToolCallRequest[], ): Promise { const targets = requests.flatMap((request) => { const receiptKey = request.receiptKey?.trim() || null; const leaseId = request.receiptLeaseId?.trim() || null; if (!receiptKey || !leaseId) return []; return [ { key: receiptKey, runId: this.currentReceiptOwnerRunId, leaseId }, ]; }); if (targets.length === 0) return; await this.markRuntimeStepReceiptsQueued(targets); } private async completeRuntimeStepReceipts( receipts: Array<{ key: string; runId: string; runAttempt?: number | null; output: unknown | null; leaseId?: string | null; }>, ): Promise> { const normalizedInputs = receipts.filter((receipt) => receipt.key.trim()); if (normalizedInputs.length === 0) return new Map(); for (const receipt of normalizedInputs) { assertNoSecretTaint(receipt.output, 'ctx.tool receipt output'); assertRuntimeReceiptOutputWithinLimit({ output: receipt.output, path: `ctx.tool receipt "${receipt.key}"`, }); } let completed: Array; try { const completeReceipts = this.#options.completeRuntimeStepReceipts; completed = completeReceipts ? await this.dispatchChunkedRuntimeReceiptRequest( normalizedInputs.map((receipt) => ({ ...receipt, runId: this.currentReceiptOwnerRunId, runAttempt: receipt.runAttempt ?? this.currentRunAttempt, })), (chunk) => completeReceipts({ receipts: chunk }), ) : await Promise.all( normalizedInputs.map((receipt) => this.completeRuntimeStepReceipt( receipt.key, receipt.runId, receipt.output, receipt.leaseId, ), ), ); } catch (error) { await this.recoverRuntimeStepReceiptCompletionsAfterBulkError( normalizedInputs, ); throw error; } const byKey = new Map(); for (let index = 0; index < completed.length; index += 1) { const normalized = this.normalizeRuntimeStepReceipt( normalizedInputs[index]?.key ?? '', completed[index], ); if (normalized) byKey.set(normalized.key, normalized); } await this.reconcileMissingRuntimeStepReceiptCompletions( normalizedInputs, byKey, ); return byKey; } private async reconcileMissingRuntimeStepReceiptCompletions( receipts: Array<{ key: string; runId: string; runAttempt?: number | null; output: unknown | null; leaseId?: string | null; }>, completedByKey: Map, ): Promise { if ( receipts.length === completedByKey.size || (!this.#options.getRuntimeStepReceipt && !this.#options.getRuntimeStepReceipts) ) { return; } const missingKeys = receipts .map((receipt) => receipt.key.trim()) .filter((key) => key && !completedByKey.has(key)); if (missingKeys.length === 0) return; const latest = await this.getRuntimeStepReceipts(missingKeys); for (const key of missingKeys) { const recovered = latest.get(key); if ( recovered?.status === 'completed' || recovered?.status === 'skipped' ) { completedByKey.set(key, recovered); } } const recoveredCount = missingKeys.filter((key) => completedByKey.has(key), ).length; if (recoveredCount > 0) { this.log( `Runtime tool receipt completion response reconciled ${recoveredCount}/${missingKeys.length} missing receipt(s) from durable store read-back.`, ); } } private async recoverRuntimeStepReceiptCompletionsAfterBulkError( receipts: Array<{ key: string; runId: string; runAttempt?: number | null; output: unknown | null; leaseId?: string | null; }>, ): Promise { if ( receipts.length === 0 || (!this.#options.getRuntimeStepReceipt && !this.#options.getRuntimeStepReceipts) ) { return; } let latest: Map; try { latest = await this.getRuntimeStepReceipts( receipts.map((receipt) => receipt.key), ); } catch { return; } await Promise.allSettled( receipts.map(async (receipt) => { const recovered = latest.get(receipt.key); if ( recovered?.status === 'completed' || recovered?.status === 'skipped' ) { return; } if ( !this.runtimeToolReceiptStillOwnedByCurrentAttempt( recovered, receipt.leaseId, ) ) { return; } await this.completeRuntimeStepReceipt( receipt.key, receipt.runId, receipt.output, receipt.leaseId, ); }), ); } private async failRuntimeStepReceipts( receipts: Array<{ key: string; runId: string; runAttempt?: number | null; error: string; leaseId?: string | null; failureKind?: WorkReceiptFailureKind | null; errorPayload?: ToolExecutionFailureV1 | null; }>, ): Promise> { const normalizedInputs = receipts.filter((receipt) => receipt.key.trim()); if (normalizedInputs.length === 0) return new Map(); const failReceipts = this.#options.failRuntimeStepReceipts; const failed = failReceipts ? await this.dispatchChunkedRuntimeReceiptRequest( normalizedInputs.map((receipt) => ({ ...receipt, runId: this.currentReceiptOwnerRunId, runAttempt: receipt.runAttempt ?? this.currentRunAttempt, })), (chunk) => failReceipts({ receipts: chunk }), ) : await Promise.all( normalizedInputs.map((receipt) => this.failRuntimeStepReceipt( receipt.key, receipt.runId, receipt.error, receipt.leaseId, receipt.failureKind, receipt.errorPayload, ), ), ); const byKey = new Map(); for (let index = 0; index < failed.length; index += 1) { const normalized = this.normalizeRuntimeStepReceipt( normalizedInputs[index]?.key ?? '', failed[index], ); if (normalized) byKey.set(normalized.key, normalized); } return byKey; } private async seedForcedRuntimeStepReceipts(keys: string[]): Promise { const receiptKeys = [ ...new Set(keys.map((key) => key.trim()).filter(Boolean)), ]; if (receiptKeys.length === 0) return; await this.claimRuntimeStepReceipts( receiptKeys, this.currentReceiptOwnerRunId, true, true, ); } private async durableToolCallCacheKeyForScope( input: { toolId: string; requestInput: Record; executionAuthScopeDigest?: string | null; staleAfterSeconds?: number | null; playLocalScope?: string | null; }, toolResponseReceiptRevision: string | null | undefined = this .currentToolResponseReceiptRevision, ): Promise { const providerActionVersion = (await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ?? ''; const executionAuthScopeDigest = input.executionAuthScopeDigest?.trim() || (await this.resolveToolAuthScopeDigest(input.toolId))?.trim() || null; return buildDurableToolCallCacheKey({ orgId: this.#options.orgId, playLocalScope: input.playLocalScope, toolId: input.toolId, requestInput: input.requestInput, authScopeDigest: buildDurableToolCallAuthScopeDigest({ orgId: this.#options.orgId, toolId: input.toolId, executionAuthScopeDigest, }), providerActionVersion, toolResponseReceiptRevision, staleAfterSeconds: input.staleAfterSeconds, cacheEpochMs: this.durableCallCacheEpochMs, }); } private async durableToolCallCacheKey( input: { toolId: string; requestInput: Record; executionAuthScopeDigest?: string | null; staleAfterSeconds?: number | null; }, toolResponseReceiptRevision: string | null | undefined = this .currentToolResponseReceiptRevision, ): Promise { return await this.durableToolCallCacheKeyForScope( { ...input, playLocalScope: this.currentGovernance.currentPlayId, }, toolResponseReceiptRevision, ); } private async resolveToolAuthScopeDigest( toolId: string, ): Promise { const configured = await this.#options.getToolAuthScopeDigest?.(toolId); if (typeof configured === 'string' && configured.trim()) { return configured.trim(); } return null; } private invalidateToolAuthScopeDigest(toolId: string): void { this.#options.invalidateToolAuthScopeDigest?.(toolId); } private durableReceiptExecutionStore(): DurableReceiptExecutionStore { return { enabled: Boolean(this.#options.claimRuntimeStepReceipt), get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey), getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys), claim: ( receiptKey, runId, reclaimRunning, forceRefresh, forceFailedRefresh, ) => this.claimRuntimeStepReceipt( receiptKey, runId, reclaimRunning, forceRefresh, forceFailedRefresh, ), ...(this.#options.markRuntimeStepReceiptRunning || this.#options.markRuntimeStepReceiptsRunning ? { markRunning: (receiptKey, runId, leaseId) => this.markRuntimeStepReceiptRunning({ key: receiptKey, runId, leaseId, }), } : {}), complete: (receiptKey, runId, output, leaseId) => this.completeRuntimeStepReceipt(receiptKey, runId, output, leaseId), release: (receiptKey, runId, leaseId) => this.releaseRuntimeStepReceipt(receiptKey, runId, leaseId), ...(this.#options.heartbeatRuntimeStepReceipts ? { heartbeat: (receiptKey, runId, leaseId) => this.heartbeatRuntimeStepReceipt(receiptKey, runId, leaseId), } : {}), fail: (receiptKey, runId, error, leaseId, failureKind, errorPayload) => this.failRuntimeStepReceipt( receiptKey, runId, error, leaseId, failureKind, errorPayload, ), canPersistFailure: Boolean(this.#options.failRuntimeStepReceipt), canPersistCompletion: Boolean( this.#options.completeRuntimeStepReceipt || this.#options.completeRuntimeStepReceipts, ), ...(this.#options.acquireRuntimeReceiptExecutionLock && this.#options.releaseRuntimeReceiptExecutionLock ? { acquireExecutionLock: ({ receiptKey, ownerExecutionId, ttlMs }) => this.#options.acquireRuntimeReceiptExecutionLock!({ key: receiptKey, runId: this.currentReceiptOwnerRunId, ownerExecutionId, ttlMs, }), releaseExecutionLock: ({ receiptKey, ownerExecutionId }) => this.#options.releaseRuntimeReceiptExecutionLock!({ key: receiptKey, runId: this.currentReceiptOwnerRunId, ownerExecutionId, }), } : {}), }; } private runtimeReceiptOutput(receipt: RuntimeStepReceipt): T { return durableRuntimeReceiptOutput(receipt); } private async waitForCompletedRuntimeToolReceipt( key: string, maxAttempts?: number, ): Promise { return await waitForCompletedRuntimeReceipt({ receiptKey: key, store: this.durableReceiptExecutionStore(), maxAttempts, toolErrorSchemaVersion: this.currentToolErrorSchemaVersion, }); } private async waitForCompletedRuntimeToolReceipts( keys: string[], maxAttempts?: number, ): Promise<{ completed: Map; failed: Map; timedOut: Set; }> { return await waitForCompletedRuntimeReceipts({ receiptKeys: keys, store: this.durableReceiptExecutionStore(), maxAttempts, log: (message) => this.log(message), toolErrorSchemaVersion: this.currentToolErrorSchemaVersion, }); } private async executeWithRuntimeReceipt( operation: DurableCtxOperation, id: string, _runId: string, opts: { force?: boolean; receiptKey?: string | null; semanticKey?: string | null; staleAfterSeconds?: number | null; repairRunningReceiptForSameRun?: boolean; repairRunningReceiptForSameRunAfterWaitTimeout?: boolean; runningReceiptWaitMaxAttempts?: number; runningReceiptWaitDelayMs?: number; reclaimRunning?: boolean; markSkipped?: (output: T) => Promise | void; onRecovered?: ( output: T, receipt: RuntimeStepReceipt, source: DurableReceiptRecoverySource, ) => T; onClaimedResult?: (output: T, receiptKey: string) => T; shouldPersistFailure?: (error: unknown) => boolean; markRunningBeforeExecute?: boolean; requiresExecutionLock?: boolean; executionLockTtlMs?: number; execute: (context: { leaseId: string | null; retainExternalCallSlot: (release: () => void) => void; }) => Promise; }, ): Promise { const stalePolicy = resolveDurableCallCachePolicy( opts.staleAfterSeconds, operation === 'step' ? 'ctx.step.staleAfterSeconds' : operation === 'fetch' ? 'ctx.fetch.staleAfterSeconds' : 'ctx.tools.execute.staleAfterSeconds', ); const receiptKey = opts.receiptKey?.trim() || durableCtxKey({ orgId: this.#options.orgId, playId: this.currentExecutionScope.receipt.namespace, operation, id, semanticKey: opts.semanticKey, staleAfterSeconds: stalePolicy.staleAfterSeconds, cacheEpochMs: this.durableCallCacheEpochMs, }); let releaseExternalCallSlot: (() => void) | null = null; try { return await executeWithDurableRuntimeReceipt({ operation, id, runId: this.currentReceiptOwnerRunId, receiptKey, store: this.durableReceiptExecutionStore(), force: opts.force === true || stalePolicy.forceRefresh, repairRunningReceiptForSameRun: opts.repairRunningReceiptForSameRun, repairRunningReceiptForSameRunAfterWaitTimeout: opts.repairRunningReceiptForSameRunAfterWaitTimeout, runningReceiptWaitMaxAttempts: opts.runningReceiptWaitMaxAttempts, runningReceiptWaitDelayMs: opts.runningReceiptWaitDelayMs, reclaimRunning: opts.reclaimRunning, markSkipped: opts.markSkipped, onRecovered: opts.onRecovered, onClaimedResult: opts.onClaimedResult, shouldPersistFailure: opts.shouldPersistFailure, markRunningBeforeExecute: opts.markRunningBeforeExecute, completedCacheOnly: operation === 'tool', withCompletedReceiptHydration: (hydrate) => this.withRuntimeReceiptHydrationTurn(hydrate), requiresExecutionLock: opts.requiresExecutionLock, executionLockTtlMs: opts.executionLockTtlMs, toolErrorSchemaVersion: this.currentToolErrorSchemaVersion, formatError: (error) => this.formatRuntimeError(error), log: (message) => this.log(message), execute: ({ leaseId }) => opts.execute({ leaseId, retainExternalCallSlot: (release) => { if (releaseExternalCallSlot) { release(); throw new Error( `ctx.${operation}(${id}) attempted to retain more than one external-call slot.`, ); } releaseExternalCallSlot = release; }, }), }); } finally { const release = releaseExternalCallSlot as (() => void) | null; release?.(); } } private get currentRunId(): string { return this.currentExecutionScope.logical.runId; } get run(): PlayAuthoringRunScope { return { id: this.currentRunId }; } private get currentReceiptOwnerRunId(): string { return this.currentExecutionScope.receipt.ownerRunId; } private get currentRunAttempt(): number { return this.currentExecutionScope.receipt.ownerAttempt; } /** * Explicit tool-node lifecycle for non-row-scoped calls (ADR 0018). Replaces * the stdout-regex inference the progress reporter used to run. */ private emitToolCallLifecycle( phase: 'started' | 'settled', toolId: string, callKey: string | null, settlement?: { outcome: 'completed' | 'no_result' | 'failed' | 'cached'; durationMs?: number | null; error?: string | null; }, ): void { if (phase === 'started') { this.emitExecutionEvent({ type: 'tool.call.started', toolId, callKey, at: Date.now(), }); return; } if (!settlement) return; this.emitExecutionEvent({ type: 'tool.call.settled', toolId, callKey, outcome: settlement.outcome, ...(settlement.durationMs !== undefined ? { durationMs: settlement.durationMs } : {}), ...(settlement.error ? { error: settlement.error } : {}), at: Date.now(), }); } /** * Direct (non-row-scoped) provider call with explicit node lifecycle. The * failure branch must emit `tool.call.settled` before rethrowing, otherwise a * failed tool node would stay `running` in the snapshot forever. */ private async executeDirectToolCall(input: { toolId: string; callKey: string; startedAt: number; input: Record; options: ToolExecutionApiOptions; }): Promise { // The caller owns settlement: it pairs started/settled in a finally so a // throw anywhere in the direct path — transport, result wrapping, receipt // ownership — settles the node exactly once instead of stranding it. return await this.callToolExecutionAPI( input.toolId, input.input, input.options, ); } /** * Bounded per-cell producer trace (ADR 0018). * * Returns the cell's accumulated attempts, with `attempt` appended when a new * one starts. Attempt records are returned by reference so a caller can * settle `outcome`/`durationMs` in place before the next patch re-states the * array. Storage is `activeMapCellMeta`, which is scoped to the running map * and released with it, so this adds no run-lifetime structure. */ /** Last recorded producer for a cell, so a later patch cannot erase it. */ private resolveCellProducer( key: string | null, fieldName: string, ): PlaySheetCellProducer | undefined { const existingCell = key ? this.activeMapCellMeta?.get(key)?.[fieldName] : undefined; const producer = existingCell && typeof existingCell === 'object' && !Array.isArray(existingCell) ? (existingCell as { producer?: unknown }).producer : undefined; return producer as PlaySheetCellProducer | undefined; } private resolveCellProducers( key: string | null, fieldName: string, attempt?: PlayCellProducerAttempt, ): { producers: PlayCellProducerAttempt[] | undefined; attemptsDropped: number; } { if (!key) { return { producers: attempt ? [attempt] : undefined, attemptsDropped: 0, }; } const existingCell = this.activeCellMetaRecord(key, fieldName); const existingProducers = existingCell?.producers; const current = Array.isArray(existingProducers) ? (existingProducers as PlayCellProducerAttempt[]) : []; const droppedSoFar = typeof existingCell?.attemptsDropped === 'number' ? existingCell.attemptsDropped : 0; if (!attempt) { return { producers: current.length > 0 ? current : undefined, attemptsDropped: droppedSoFar, }; } const next = [...current, attempt]; if (next.length <= MAX_CELL_PRODUCER_ATTEMPTS) { return { producers: next, attemptsDropped: droppedSoFar }; } // Keep the newest attempts — a cascade's decisive legs are its last ones — // but count what fell off. Without the count a capped cell's logicalCalls // reads as a complete total when it is a floor. const dropped = next.length - MAX_CELL_PRODUCER_ATTEMPTS; return { producers: next.slice(dropped), attemptsDropped: droppedSoFar + dropped, }; } /** * Carry this run's durable producer trace into a re-executed row (ADR 0018). * * `activeMapCellMeta` starts empty on a worker replay, and the durable jsonb * merge replaces a cell's `producers` array wholesale, so a re-executed cell * would erase the attempts the earlier execution recorded — silently * under-counting `logicalCalls` and losing the only record a failed leg ever * gets. The row already arrives carrying its durable cell meta (it is what * `previousCell` is built from), so seeding costs no extra round trip. * * Only cells stamped with the CURRENT run are seeded. A previous run's * attempts are that run's facts; appending them here would inflate this run. * Seeding read-modify-writes nothing durable: the append still lands through * the attempt-fenced write, so a fenced-out attempt cannot corrupt the trace. */ private seedActiveCellProducersFromDurableRow( rowKey: string | null, baseRow: Record, ): void { if (!rowKey || !this.activeMapCellMeta) return; const durable = baseRow[DEEPLINE_CELL_META_FIELD]; if (!durable || typeof durable !== 'object' || Array.isArray(durable)) { return; } for (const [fieldName, rawCell] of Object.entries( durable as Record, )) { if (!rawCell || typeof rawCell !== 'object' || Array.isArray(rawCell)) { continue; } const cell = rawCell as Record; if (cell.runId !== this.currentRunId) continue; if (!Array.isArray(cell.producers) || cell.producers.length === 0) { continue; } const existingRow = this.activeMapCellMeta.get(rowKey); const existingCell = this.activeCellMetaRecord(rowKey, fieldName); if (existingCell?.producers !== undefined) continue; this.activeMapCellMeta.set(rowKey, { ...(existingRow ?? {}), [fieldName]: { ...(existingCell ?? {}), producers: cell.producers, ...(typeof cell.attemptsDropped === 'number' ? { attemptsDropped: cell.attemptsDropped } : {}), }, }); } } /** Existing `_cell_meta` record for a cell inside the running map, if any. */ private activeCellMetaRecord( key: string | null, fieldName: string, ): Record | null { const existingCell = key ? this.activeMapCellMeta?.get(key)?.[fieldName] : undefined; return existingCell && typeof existingCell === 'object' && !Array.isArray(existingCell) ? (existingCell as Record) : null; } /** * Provenance capture is a field on writes that already happen (ADR 0019), so * a record made *before* the cell's next patch is seeded into the running * map's cell meta and re-stated by that patch. No extra emit, no extra round * trip, and the record is released with the map like the producer trace. */ private recordCellProvenance( key: string | null, fieldName: string, record: { reads?: PlayCellReadRef[]; decide?: PlayCellDecision }, ): void { if (!key || !this.activeMapCellMeta) return; if (record.reads === undefined && record.decide === undefined) return; const existingRow = this.activeMapCellMeta.get(key); const existingCell = this.activeCellMetaRecord(key, fieldName); this.activeMapCellMeta.set(key, { ...(existingRow ?? {}), [fieldName]: { ...(existingCell ?? {}), ...(record.reads !== undefined ? { reads: record.reads } : {}), ...(record.decide !== undefined ? { decide: record.decide } : {}), }, }); } /** Row-grain read order for the running map's row, if one was recorded. */ private resolveRowReadOrder( key: string | null, ): PlayRowReadOrder | undefined { const rowMeta = this.activeCellMetaRecord(key, ROW_META_CELL_KEY); const reads = rowMeta?.reads; return reads && typeof reads === 'object' && !Array.isArray(reads) ? (reads as PlayRowReadOrder) : undefined; } /** * Open one cell against the row's read order (ADR 0019). * * The order lives once per row under the reserved `_row` key instead of once * per cell. A written column is appended before its own cell runs, so its * position in the list *is* its cut point — everything before it was * available to it, itself and everything after it was not. That makes the * common case a single string append with no per-cell record at all. Cells * the list cannot place (non-persisted fields, and anything past the column * cap) fall back to an explicit cut, and cells past the cell cap are counted * as dropped rather than silently reading as having read nothing. */ private recordRowReadCell( key: string | null, fieldName: string, persisted: boolean, ): void { if (!key || !this.activeMapCellMeta) return; const order = this.resolveRowReadOrder(key); if (!order || order.columns.length === 0) return; if (cellReadCut(order, fieldName) !== null) return; if (persisted && order.columns.length < MAX_CELL_READ_REFS) { this.mergeRowMeta(key, { reads: { ...order, columns: [...order.columns, fieldName] }, }); return; } const upto = order.upto ?? {}; if (Object.keys(upto).length >= MAX_ROW_READ_CELLS) { this.mergeRowMeta(key, { reads: { ...order, droppedCells: (order.droppedCells ?? 0) + 1 }, }); return; } this.mergeRowMeta(key, { reads: { ...order, upto: { ...upto, [fieldName]: order.columns.length } }, }); } /** Seed the row's read order with the columns the row arrived carrying. */ private recordRowReadColumns( key: string | null, columns: readonly string[], ): void { if (!key || !this.activeMapCellMeta || columns.length === 0) return; const order = this.resolveRowReadOrder(key); const merged = order ? [...order.columns] : []; for (const column of columns) { if (merged.length >= MAX_CELL_READ_REFS) break; if (merged.includes(column)) continue; merged.push(column); } if (order && merged.length === order.columns.length) return; this.mergeRowMeta(key, { reads: { ...(order ?? {}), columns: merged }, }); } /** Merge a row-grain record under the reserved `_row` cell-meta key. */ private mergeRowMeta(key: string, record: PlayRowMeta): void { if (!this.activeMapCellMeta) return; const existingRow = this.activeMapCellMeta.get(key); const existingMeta = this.activeCellMetaRecord(key, ROW_META_CELL_KEY); this.activeMapCellMeta.set(key, { ...(existingRow ?? {}), [ROW_META_CELL_KEY]: { ...(existingMeta ?? {}), ...record }, }); } /** * Explicit per-cell read array, re-stated so a later patch cannot erase it. * * Only nested step-program cells carry one: their read-set includes sibling * steps, which are not part of the row's column order, so the row-grain cut * point cannot express it. Dataset columns resolve through the row order. */ private resolveCellReads( key: string | null, fieldName: string, ): PlayCellReadRef[] | undefined { const reads = this.activeCellMetaRecord(key, fieldName)?.reads; return Array.isArray(reads) ? (reads as PlayCellReadRef[]) : undefined; } /** The read-set a cell saw, from its explicit array or the row order. */ private effectiveCellReads( key: string | null, fieldName: string, ): PlayCellReadRef[] { return ( this.resolveCellReads(key, fieldName) ?? hydrateCellReadRefs(this.resolveRowReadOrder(key), fieldName, '')?.map( (ref) => ({ column: ref.column }), ) ?? [] ); } /** Last recorded branch decision for a cell, re-stated on every patch. */ private resolveCellDecision( key: string | null, fieldName: string, ): PlayCellDecision | undefined { const decide = this.activeCellMetaRecord(key, fieldName)?.decide; return decide && typeof decide === 'object' && !Array.isArray(decide) ? (decide as PlayCellDecision) : undefined; } /** * Bounded read-set for one cell (ADR 0019). * * The runtime hands a column resolver the whole row, so the read-set it * witnesses is the row's columns at eval time — not a per-property access * trace. Recording the resolved column list as-is is the honest form: it is a * superset of what the resolver touched, and it is an observation rather than * a declaration. Per-property precision needs sandbox proxy tracking, which * is deliberately out of this wave. * * Only enumerable columns are recorded. CSV alias projections are * non-enumerable by construction, so they read as absent rather than as a * column the runtime can name. * * Refs carry no `table` or `rowKey` because every read here is same-row and * same-table; the reader fills both in from the cell's own address. Writing * the row key into each ref put a mapped row over the retained-row memory * budget, and the budget is not the thing that moves. */ private cellReadRefsForRow( rowKey: string | null, rowSources: readonly Record[], targetFieldName: string, /** * Columns this map will write but has not written yet in this run. On a * rerun the row arrives carrying their previous values, which is * `previousCell` — the same fact the self-exclusion below rejects, and * recording it would manufacture a cycle between two columns that each * saw the other's stale value. */ pendingOutputColumns?: ReadonlySet, ): PlayCellReadRef[] | undefined { if (!rowKey) return undefined; const refs: PlayCellReadRef[] = []; const seen = new Set(); for (const source of rowSources) { for (const column of Object.keys(source)) { if (refs.length >= MAX_CELL_READ_REFS) break; // A cell never reads itself: on a rerun the target column is present in // the row carrying the *previous* run's value, which is a different // fact (`previousCell`), not an input to this computation. if (column === targetFieldName) continue; if (pendingOutputColumns?.has(column)) continue; if (!shouldPersistMapCellField(column)) continue; if (column.startsWith('__deepline')) continue; if (seen.has(column)) continue; seen.add(column); refs.push({ column }); } } return refs.length > 0 ? refs : undefined; } /** * The row's input columns — everything the first cell of the row could read. * Columns this map writes are excluded even when the row already carries a * value for them, because on a rerun that value is the previous run's. */ private rowInputReadColumns( baseRow: Record, datasetColumns: ReadonlySet, ): string[] { const columns: string[] = []; for (const column of Object.keys(baseRow)) { if (columns.length >= MAX_CELL_READ_REFS) break; if (datasetColumns.has(column)) continue; if (!shouldPersistMapCellField(column)) continue; if (column.startsWith('__deepline')) continue; columns.push(column); } return columns; } private emitScopedFieldMetaUpdate(input: { rowId: number; key: string | null; tableNamespace: string | null; fieldName?: string | null; status: | 'queued' | 'running' | 'completed' | 'failed' | 'cached' | 'missed' | 'skipped'; rowStatus?: PlayRowUpdate['status']; stage?: string | null; provider?: string | null; error?: string | null; reused?: boolean; completedAt?: number; staleAt?: number | null; staleAfterSeconds?: number | null; producer?: { kind: 'play' | 'tool' | 'code'; id?: string | null; displayName?: string | null; playId?: string | null; toolId?: string | null; runId?: string | null; } | null; /** * A newly started producer attempt for this cell (ADR 0018). The record is * appended to the cell's bounded `producers` trace and kept by reference so * the caller can settle its outcome/duration in place. */ producerAttempt?: PlayCellProducerAttempt; dataPatch?: Record; }): void { if (!input.fieldName) { this.emitScopedRowUpdate(input.key, input.tableNamespace, { rowId: input.rowId, status: input.rowStatus, stage: input.stage ?? null, provider: input.provider ?? null, error: input.error ?? null, dataPatch: input.dataPatch ?? {}, }); return; } // Every patch for a cell re-states its producer trace. The in-memory cell // patch merge replaces a field wholesale, so carrying the array forward is // what keeps a waterfall's losing legs alive through the cell's completion // patch (which deliberately omits `producer`). // // Everything from here to `rowReads` is the attribution capture the docflow // campaign added (ADR 0018 producers, ADR 0019 reads/decide). Resolved as a // block behind the run's rollout answer so that an ungated run emits the // cell patch main emitted, key for key — including OMITTING `producer` when // the caller passed none, which is the one pre-existing key whose behavior // the re-statement rule changed. const capture = this.docflowCaptureEnabled; const { producers, attemptsDropped } = capture ? this.resolveCellProducers( input.key, input.fieldName, input.producerAttempt, ) : { producers: undefined, attemptsDropped: 0 }; // The durable jsonb merge preserves keys a patch omits; the in-memory cell // patch merge replaces the field wholesale. Re-state the winning producer // so the cell's terminal write does not erase it (ADR 0018). const producer = input.producer !== undefined ? input.producer : capture ? this.resolveCellProducer(input.key, input.fieldName) : undefined; // Same re-statement discipline for the provenance capture (ADR 0019): both // were seeded before the cell resolved, and the wholesale field replace // would otherwise drop them on the cell's terminal patch. const reads = capture ? this.resolveCellReads(input.key, input.fieldName) : undefined; const decide = capture ? this.resolveCellDecision(input.key, input.fieldName) : undefined; // The row's read order is row-grain, so it rides the patch once under the // reserved key rather than being copied onto every cell. const rowReads = capture ? this.resolveRowReadOrder(input.key) : undefined; this.emitScopedRowUpdate(input.key, input.tableNamespace, { rowId: input.rowId, status: input.rowStatus, stage: input.stage ?? null, provider: input.provider ?? null, error: input.error ?? null, dataPatch: input.dataPatch ?? {}, ...(rowReads ? { rowMetaPatch: { reads: rowReads } } : {}), cellMetaPatch: { [input.fieldName]: { status: input.status, stage: input.stage ?? null, provider: input.provider ?? null, error: input.error ?? null, ...(producer !== undefined ? { producer } : {}), // Only attribution-bearing cells carry the run stamp and the trace. // Both ride the retained-row payload, so paying for them on every // status-only cell patch would move the map memory budget. ...(producers !== undefined ? { runId: this.currentRunId, producers, // Loud under-count: a capped trace states what it lost. ...(attemptsDropped > 0 ? { attemptsDropped } : {}), } : {}), ...(reads !== undefined ? { reads } : {}), ...(decide !== undefined ? { decide } : {}), ...(input.reused !== undefined ? { reused: input.reused } : {}), ...(input.completedAt !== undefined ? { completedAt: input.completedAt } : {}), ...(input.staleAt !== undefined ? { staleAt: input.staleAt } : {}), ...(input.staleAfterSeconds !== undefined ? { staleAfterSeconds: input.staleAfterSeconds } : {}), }, }, }); } private isCompletedFieldValue(value: unknown): boolean { return ( value !== null && value !== undefined && !(typeof value === 'string' && value.length === 0) ); } // --- Runtime values survive the dataset persist/resume boundary --- // Tool results and PlayDatasets carry live methods. Cells store JSON, so // encode them before persistence and revive them before authored play code // receives a row again. private async serializeCellValue(value: unknown): Promise { const serialized = isToolExecuteResult(value) ? serializeToolExecuteResult(value) : await this.serializeDatasetCells(value); try { stringifyPostgresJson(serialized); } catch (error) { throw new Error( `${RUNTIME_ROW_OUTPUT_NOT_JSON_SERIALIZABLE}: Dataset cell output is not JSON-serializable: ${ error instanceof Error ? error.message : String(error) }`, ); } return serialized; } private async serializeDatasetCells(value: unknown): Promise { // Keep cycles intact for stringifyPostgresJson to reject. This pass only // encodes runtime values; it must not turn an invalid cyclic cell into a // different graph or recurse forever before the JSON boundary can report it. const visiting = new WeakSet(); const visit = async (current: unknown): Promise => { if (isToolExecuteResult(current)) { return serializeToolExecuteResult(current); } if (isPlayDataset(current)) { return await serializePlayDatasetCell(current, visit); } if (!current || typeof current !== 'object') return current; if (visiting.has(current)) return current; visiting.add(current); try { if (Array.isArray(current)) { let copy: unknown[] | null = null; for (const [index, entry] of current.entries()) { const serialized = await visit(entry); if (serialized !== entry) { copy ??= [...current]; copy[index] = serialized; } } return copy ?? current; } if (!isPlainRecord(current)) return current; let copy: Record | null = null; for (const [key, entry] of Object.entries(current)) { const serialized = await visit(entry); if (serialized !== entry) { copy ??= { ...current }; copy[key] = serialized; } } return copy ?? current; } finally { visiting.delete(current); } }; return await visit(value); } private rehydrateCellValue(value: unknown): unknown { const seen = new WeakMap(); const visit = (current: unknown): unknown => { if (isToolExecuteResult(current)) { return current; } if (isSerializedToolExecuteResult(current)) { return deserializeToolExecuteResult(current); } if (isSerializedPlayDatasetCell(current)) { return deserializePlayDatasetCell(current, visit); } if ( isSerializedPlayDataset(current) && current.datasetId.startsWith('tool-list:') ) { return deserializeLegacyPlayDataset(current); } if (!current || typeof current !== 'object') return current; const existing = seen.get(current); if (existing !== undefined) return existing; if (Array.isArray(current)) { const copy: unknown[] = []; seen.set(current, copy); for (const entry of current) copy.push(visit(entry)); return copy; } if (!isPlainRecord(current)) return current; const copy: Record = {}; seen.set(current, copy); for (const [key, entry] of Object.entries(current)) { copy[key] = visit(entry); } return copy; }; return visit(value); } /** * Rehydrate serialized tool-execute results in a row before it is handed to a * column resolver. The row is always a fresh `cloneCsvAliasedRow` clone, so * mutating its enumerable fields in place is safe and preserves the * non-enumerable CSV-projection metadata. */ private rehydrateRowFields( row: Record, ): Record { for (const key of Object.keys(row)) { row[key] = this.rehydrateCellValue(row[key]); } return row; } private previousCellForField( row: Record, fieldName: string, ): PreviousCell | undefined { const hasValue = Object.prototype.hasOwnProperty.call(row, fieldName) && this.isCompletedFieldValue(row[fieldName]); return previousCellFromValue({ hasValue, value: this.rehydrateCellValue(row[fieldName]), meta: this.cellMetaForField(row, fieldName), }); } private cellMetaForField(row: Record, fieldName: string) { const cellMeta = row[DEEPLINE_CELL_META_FIELD] && typeof row[DEEPLINE_CELL_META_FIELD] === 'object' ? (row[DEEPLINE_CELL_META_FIELD] as Record) : {}; const rawMeta = cellMeta[fieldName]; return rawMeta && typeof rawMeta === 'object' ? (rawMeta as { status?: string; completedAt?: number; staleAt?: number | null; staleAfterSeconds?: number | null; }) : null; } private toVisibleDataPatch( fields: Record, ): Record { return Object.fromEntries( Object.entries(fields).filter(([fieldName]) => shouldPersistMapCellField(fieldName), ), ); } private formatRuntimeError(error: unknown): string { if (error instanceof Error) { return error.message; } return String(error); } private effectiveToolCallCachePolicy(options?: ToolCallOptions): { force: boolean; forceFailedRefresh: boolean; staleAfterSeconds?: number | null; } { const stalePolicy = resolveDurableCallCachePolicy( options?.staleAfterSeconds, ); return { force: options?.force === true || stalePolicy.forceRefresh || this.#options.cachePolicy?.forceToolRefresh === true, forceFailedRefresh: options?.force === true || stalePolicy.forceRefresh || this.#options.cachePolicy?.forceToolRefresh === true || this.#options.cachePolicy?.forceFailedToolRefresh === true, staleAfterSeconds: stalePolicy.staleAfterSeconds, }; } private summarizeBatchSizes(sizes: readonly number[]): string { if (sizes.length <= BATCH_SIZE_LOG_SAMPLE_LIMIT) { return sizes.join(', '); } const sample = sizes.slice(0, BATCH_SIZE_LOG_SAMPLE_LIMIT).join(', '); const remaining = sizes.length - BATCH_SIZE_LOG_SAMPLE_LIMIT; return `${sample}, ... +${remaining} more`; } private getCachedToolResult( toolId: string, rowCacheKey: string, path: 'mapped' | 'direct' = 'mapped', ): ToolBatchResult | undefined { if ( path === 'direct' ? this.durableDirectToolResultsBackedByReceipts : this.durableMappedToolResultsBackedByReceipts ) { return undefined; } return this.checkpoint.completedToolBatches[toolId]?.[rowCacheKey]; } private getCachedToolResultCandidate( toolId: string, rowCacheKeys: readonly string[], path: 'mapped' | 'direct' = 'mapped', ): { cacheKey: string; result: ToolBatchResult } | null { for (const rowCacheKey of rowCacheKeys) { const cached = this.getCachedToolResult(toolId, rowCacheKey, path); if (cached?.done) { return { cacheKey: rowCacheKey, result: cached }; } } return null; } private cacheToolResult( toolId: string, rowCacheKey: string, result: unknown | null, path: 'mapped' | 'direct' = 'mapped', ): void { if ( path === 'direct' ? this.durableDirectToolResultsBackedByReceipts : this.durableMappedToolResultsBackedByReceipts ) { return; } if (!this.checkpoint.completedToolBatches[toolId]) { this.checkpoint.completedToolBatches[toolId] = {}; } this.checkpoint.completedToolBatches[toolId]![rowCacheKey] = { done: true, result, }; } private buildToolResultCacheKey(input: { rowId: number; tableNamespace?: string; rowKey?: string; callId?: string; }): string { const scope = this.currentGovernance.currentRunId || this.#options.runId || 'run'; if (input.callId?.trim()) { return `${scope}:${input.callId.trim()}`; } if (input.rowKey?.trim()) { return `${scope}:${input.rowKey.trim()}`; } if (input.tableNamespace?.trim()) { return `${scope}:${input.tableNamespace.trim()}:${input.rowId}`; } return `${scope}:direct:${input.rowId}`; } private async resolveToolResultMetadata( toolId: string, ): Promise { const metadata = await this.#options.getToolResultMetadata?.(toolId); return { toolId, extractors: metadata?.extractors ?? {}, targetGetters: metadata?.targetGetters ?? {}, listExtractorPaths: metadata?.listExtractorPaths ?? [], listIdentityGetters: metadata?.listIdentityGetters ?? {}, }; } private async wrapToolExecutionResult(input: { toolId: string; status: string; jobId?: string; result: unknown; metadata?: ToolResultMetadataInput | null; meta?: Record; toolResponse?: { raw?: unknown; rawV2?: unknown; view?: 'data' | 'rawV2'; meta?: Record; }; execution: ToolResultExecutionMetadata; requestInput?: Record; }): Promise { if (isToolExecuteResult(input.result)) { return this.attachCustomerDbDatasetResult( input.toolId, input.requestInput, cloneToolExecuteResultWithExecution(input.result, input.execution), ); } const publicToolResult = publicToolResponseEnvelope(input.result); return this.attachCustomerDbDatasetResult( input.toolId, input.requestInput, createToolExecuteResult({ status: publicToolResult?.status ?? input.status, jobId: input.jobId, result: publicToolResult ? { data: publicToolResult.raw, ...(publicToolResult.meta ? { meta: publicToolResult.meta } : {}), } : input.result, response: publicToolResult ? { ...(Object.prototype.hasOwnProperty.call( publicToolResult, 'rawV2', ) ? { rawV2: publicToolResult.rawV2 } : {}), ...(publicToolResult.view ? { view: publicToolResult.view } : {}), ...(publicToolResult.meta ? { meta: publicToolResult.meta } : {}), } : input.toolResponse, metadata: input.metadata ?? (await this.resolveToolResultMetadata(input.toolId)), execution: input.execution, meta: input.meta, }), ); } private attachCustomerDbDatasetResult( toolId: string, requestInput: Record | undefined, wrapped: ToolExecuteResult, ): ToolExecuteResult { if (!isQueryResultDatasetTool(toolId)) return wrapped; const raw = recordOrNull(wrapped.toolResponse.raw); const dataset = recordOrNull(raw?.dataset); const rows = rowsFromUnknown(raw?.rows); const totalRows = finiteNonNegativeInteger(dataset?.total_rows); const sql = typeof requestInput?.sql === 'string' ? requestInput.sql : typeof requestInput?.query === 'string' ? requestInput.query : typeof raw?.sql === 'string' ? raw.sql : null; if (!dataset || totalRows === null || !sql) { return wrapped; } const originalRequestInput = requestInput as Record; const datasetLimit = finitePositiveInteger(dataset.returned_limit) ?? totalRows; const effectiveCount = Math.min(totalRows, datasetLimit); const previewRows = rows.slice(0, Math.min(rows.length, 25)); const executionNonce = typeof wrapped.job_id === 'string' && wrapped.job_id.trim() ? wrapped.job_id.trim() : stableDigest( `${this.currentRunId}:${toolId}:${stableStringify(originalRequestInput)}:${datasetLimit}`, ); const datasetId = `tool-list:${sha256Hex( `${toolId}:${this.currentRunId}:${sql}:${datasetLimit}:${executionNonce}`, )}`; const fetchPage = async ( offset: number, limit: number, ): Promise[]> => { if (limit <= 0 || offset >= totalRows) return []; const execution = await this.callToolExecutionAPI( toolId, originalRequestInput, { durableCallReceiptKey: `${buildDurableToolReceiptPrefix({ orgId: this.#options.orgId ?? 'unknown-org', toolId, })}${stableDigest(`${datasetId}:${offset}:${limit}`)}`, timeoutMs: resolveToolRuntimeTimeoutMs(toolId), customerDbDataset: { limit: datasetLimit, offset, pageSize: Math.min(limit, QUERY_RESULT_DATASET_PAGE_SIZE), totalRows, }, }, ); const pageRaw = recordOrNull(execution.toolResponse)?.raw ?? recordOrNull(execution.result)?.data ?? execution.result; return rowsFromUnknown(recordOrNull(pageRaw)?.rows); }; const collectRows = async ( limit: number | undefined, ): Promise[]> => { const target = Math.min(limit ?? totalRows, totalRows, datasetLimit); const collected: Record[] = []; for ( let offset = 0; offset < target; offset += QUERY_RESULT_DATASET_PAGE_SIZE ) { collected.push( ...(await fetchPage( offset, Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, target - offset), )), ); } return collected.slice(0, target); }; const playDataset = createDeferredPlayDataset({ datasetKind: 'csv', datasetId, count: effectiveCount, previewRows, residentRows: rows.length >= effectiveCount ? rows.slice(0, effectiveCount) : null, sourceLabel: 'query result rows', tableNamespace: null, resolvers: { count: async () => Math.min(totalRows, datasetLimit), at: async (index) => (await fetchPage(index, 1))[0], peek: async (limit) => collectRows(limit), materialize: async (limit) => collectRows(limit), iterate: () => ({ async *[Symbol.asyncIterator]() { const count = Math.min(totalRows, datasetLimit); for ( let offset = 0; offset < count; offset += QUERY_RESULT_DATASET_PAGE_SIZE ) { yield* await fetchPage( offset, Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, count - offset), ); } }, }) as AsyncIterable>, }, }); return attachToolResultListDataset(wrapped, { name: 'rows', path: 'toolResponse.raw.rows', dataset: playDataset, count: Math.min(totalRows, datasetLimit), }); } private async resolveToolCall( toolId: string, request: ToolCallRequest, result: unknown | null, metadata?: ToolResultMetadataInput | null, jobId?: string, meta?: Record, toolResponse?: ParsedToolExecuteResponse['toolResponse'], ): Promise { const cacheKey = request.cacheKey; const receiptKey = request.receiptKey?.trim() || null; const wrapped = await this.wrapToolExecutionResult({ toolId, status: result == null ? 'no_result' : 'completed', jobId, result, metadata, meta, toolResponse, execution: toolExecutionMetadataForOutcome({ kind: 'live', cacheKey, receiptKey, }), requestInput: request.input, }); const releaseCompletionHold = this.holdRuntimeToolReceiptCompletions( receiptKey ? [receiptKey] : [], ); try { let completed: RuntimeStepReceipt | null | undefined = null; if (receiptKey) { try { completed = ( await this.completeRuntimeStepReceipts([ { key: receiptKey, runId: this.currentReceiptOwnerRunId, leaseId: request.receiptLeaseId ?? null, output: serializeToolExecuteResult(wrapped), }, ]) ).get(receiptKey); } catch (receiptError) { // Completion failed AFTER a successful (billed) tool call: trip the // breaker so remaining rows stop dispatching new provider calls. tripRuntimePersistenceLatch(this.persistenceLatch, receiptError); throw receiptError; } } return await this.finalizeResolvedToolCall( toolId, request, wrapped, completed, ); } finally { releaseCompletionHold(); } } private async resolveToolCallBatchResults( toolId: string, entries: Array<{ request: ToolCallRequest; result: unknown | null; status?: string; metadata?: ToolResultMetadataInput | null; jobId?: string; meta?: Record; toolResponse?: ParsedToolExecuteResponse['toolResponse']; }>, ): Promise { const wrappedEntries = await Promise.all( entries.map(async (entry) => ({ ...entry, wrapped: await this.wrapToolExecutionResult({ toolId, status: entry.status ?? (entry.result == null ? 'no_result' : 'completed'), jobId: entry.jobId, result: entry.result, metadata: entry.metadata, meta: entry.meta, toolResponse: entry.toolResponse, execution: toolExecutionMetadataForOutcome({ kind: 'live', cacheKey: entry.request.cacheKey, receiptKey: entry.request.receiptKey, }), requestInput: entry.request.input, }), })), ); const receiptInputs = wrappedEntries.flatMap((entry) => { const receiptKey = entry.request.receiptKey?.trim() || null; return receiptKey ? [ { key: receiptKey, runId: this.currentReceiptOwnerRunId, leaseId: entry.request.receiptLeaseId ?? null, output: serializeToolExecuteResult(entry.wrapped), }, ] : []; }); const releaseCompletionHold = this.holdRuntimeToolReceiptCompletions( receiptInputs.map((receipt) => receipt.key), ); try { let completedByKey = new Map(); if (receiptInputs.length > 0) { try { completedByKey = await this.completeRuntimeStepReceipts(receiptInputs); } catch (receiptError) { // Completion failed AFTER successful (billed) tool calls: trip the // breaker so remaining rows stop dispatching new provider calls. tripRuntimePersistenceLatch(this.persistenceLatch, receiptError); await Promise.all( wrappedEntries.map((entry) => this.rejectToolCall(toolId, entry.request, receiptError, { persistReceiptFailure: false, }), ), ); throw receiptError; } } const results: unknown[] = []; for (const entry of wrappedEntries) { const receiptKey = entry.request.receiptKey?.trim() || null; results.push( await this.finalizeResolvedToolCall( toolId, entry.request, entry.wrapped, receiptKey ? completedByKey.get(receiptKey) : null, ), ); } return results; } finally { releaseCompletionHold(); } } private holdRuntimeToolReceiptCompletions(receiptKeys: string[]): () => void { const uniqueKeys = [ ...new Set(receiptKeys.map((key) => key.trim())), ].filter(Boolean); for (const key of uniqueKeys) { this.runtimeToolReceiptCompletionsInFlight.set( key, (this.runtimeToolReceiptCompletionsInFlight.get(key) ?? 0) + 1, ); } let released = false; return () => { if (released) return; released = true; for (const key of uniqueKeys) { const count = this.runtimeToolReceiptCompletionsInFlight.get(key) ?? 0; if (count <= 1) { this.runtimeToolReceiptCompletionsInFlight.delete(key); } else { this.runtimeToolReceiptCompletionsInFlight.set(key, count - 1); } } }; } private async finalizeResolvedToolCall( toolId: string, request: ToolCallRequest, wrapped: ToolExecuteResult, completed: RuntimeStepReceipt | null | undefined, ): Promise { const cacheKey = request.cacheKey; const receiptKey = request.receiptKey?.trim() || null; const canPersistReceiptCompletion = Boolean( this.#options.completeRuntimeStepReceipt || this.#options.completeRuntimeStepReceipts, ); if ( receiptKey && canPersistReceiptCompletion && completed?.status !== 'completed' && completed?.status !== 'skipped' ) { this.log( `Durable tool call ${receiptKey} completed live after its receipt lease moved; using live result without overwriting the receipt.`, ); } const finalWrapped = (completed?.status === 'completed' || completed?.status === 'skipped') && completed.output !== undefined ? await this.wrapToolExecutionResult({ toolId, status: completed.output === null || completed.output === undefined ? 'no_result' : 'completed', result: this.runtimeReceiptOutput(completed), requestInput: request.input, execution: toolExecutionMetadataForOutcome( completed.runId === this.currentReceiptOwnerRunId ? { kind: 'live', cacheKey, receiptKey: request.receiptKey, } : { kind: 'cache', cacheKey, receiptKey: receiptKey ?? '', attachedToReceiptKey: receiptKey, }, ), }) : wrapped; if (request.cacheable !== false) { this.cacheToolResult(toolId, cacheKey, finalWrapped); } const resolver = this.toolCallResolvers.get(request.callId); if (resolver) { resolver.resolve(finalWrapped); this.toolCallResolvers.delete(request.callId); } this.emitScopedFieldMetaUpdate({ rowId: request.rowId, key: request.rowKey ?? null, tableNamespace: request.tableNamespace ?? null, fieldName: request.fieldName, status: 'running', rowStatus: 'running', stage: toolId, provider: null, error: null, dataPatch: {}, }); return finalWrapped; } private async rejectToolCall( toolId: string, request: ToolCallRequest, error: unknown, options?: { persistReceiptFailure?: boolean }, ): Promise { const message = this.formatRuntimeError(error); let rejectionError = error; const receiptKey = request.receiptKey?.trim() || null; if (receiptKey && options?.persistReceiptFailure !== false) { const persistenceAlreadyFailed = this.persistenceLatch.tripped; try { await this.failRuntimeStepReceipts([ { key: receiptKey, runId: this.currentReceiptOwnerRunId, leaseId: request.receiptLeaseId ?? null, error: message, errorPayload: serializeToolExecutionFailure(error), failureKind: runtimeReceiptFailureKindForError(error), }, ]); } catch (receiptError) { tripRuntimePersistenceLatch(this.persistenceLatch, receiptError); if (!persistenceAlreadyFailed) { const receiptFailureMessage = this.formatRuntimeError(receiptError); rejectionError = new AggregateError( [error, receiptError], `Tool call failed and durable receipt could not be marked failed: ${message}; receipt failure: ${receiptFailureMessage}`, ); } } } const resolver = this.toolCallResolvers.get(request.callId); if (resolver) { resolver.reject( rejectionError instanceof Error ? rejectionError : new Error(message), ); this.toolCallResolvers.delete(request.callId); } this.emitScopedFieldMetaUpdate({ rowId: request.rowId, key: request.rowKey ?? null, tableNamespace: request.tableNamespace ?? null, fieldName: request.fieldName, status: 'failed', rowStatus: 'running', stage: toolId, provider: null, error: message, dataPatch: {}, }); } private pulseProgressHeartbeat(force = false): void { if (!this.#options.onBatchComplete) { return; } const now = Date.now(); if ( !force && now - this.lastProgressHeartbeatAt < PROGRESS_HEARTBEAT_INTERVAL_MS ) { return; } this.lastProgressHeartbeatAt = now; this.#options.onBatchComplete(this.checkpoint); } // ——— Public ctx API ——— async csv( path: string, options?: CsvOptions, ): Promise>> { this.assertInlineChildContract('dataset_child'); if (options) { for (const [field, value] of [ ['ctx.csv.options.description', options.description], ['ctx.csv.options.columns', options.columns], ['ctx.csv.options.rename', options.rename], ['ctx.csv.options.required', options.required], ] as const) { validateOptionalPlayAuthoringField(field, value); } } // In cloud mode, CSV data is passed in — path is just a label // The activity loads the actual data before creating the ctx throw new Error( `ctx.csv("${path}") is handled by the workflow. ` + `CSV data is loaded by the runtime and passed to your play via input.`, ); } dataset>( key: string, items: PlayDatasetInput, ): RuntimeDatasetBuilder; dataset>( key: string, items: PlayDatasetInput, input: RuntimeStepProgram, options?: RuntimeDatasetOptions, ): Promise>>; dataset< T extends Record, TColumns extends Record = Record, >( key: string, items: PlayDatasetInput, input?: MapFieldDefinition | RuntimeStepProgram, options?: RuntimeDatasetOptions, ): RuntimeDatasetBuilder | Promise>> { this.assertInlineChildContract('dataset_child'); if (rowContext.getStore()) { throw new Error( 'Nested ctx.dataset() is not supported. Flatten your fields into one dataset, or keep custom per-row logic inside a single column.', ); } if (input === undefined) { return new RuntimeDatasetBuilder(this, key, items); } if (isRuntimeStepProgram(input)) { return this.runStepProgramMap(key, items, input, options); } throw new Error( 'ctx.dataset(key, rows, fields, options) is not supported. Use ctx.dataset(key, rows).withColumn(...).run(options).', ); } map>( key: string, items: PlayDatasetInput, ): never; map>( key: string, items: PlayDatasetInput, input: RuntimeStepProgram, options?: RuntimeDatasetOptions, ): never; map>( _key: string, _items: PlayDatasetInput, _input?: | MapFieldDefinition> | RuntimeStepProgram, _options?: RuntimeDatasetOptions, ): never { void _key; void _items; void _input; void _options; throw new Error(CTX_MAP_MIGRATION_MESSAGE); } async runStepProgramMap>( key: string, items: PlayDatasetInput, program: RuntimeStepProgram, options?: RuntimeDatasetOptions, ): Promise>> { const definition = this.stepProgramToMapDefinition(program); return this.runMapDefinition(key, items, definition, options); } async runSteps, TOutput = unknown>( program: RuntimeStepProgram, input: TInput, options?: { description?: string }, ): Promise { if (!isRuntimeStepProgram(program)) { throw new Error('ctx.runSteps(program, input) requires steps().'); } if (options?.description !== undefined) { validatePlayAuthoringField( 'ctx.runSteps.options.description', options.description, ); this.log(options.description); } return (await this.executeStepProgram(program, input, 0, [], { checkpointSteps: true, })) as TOutput; } private async runMapDefinition< T extends Record, TColumns extends Record = Record, >( key: string, items: PlayDatasetInput, input: MapFieldDefinition, options?: RuntimeDatasetOptions, ): Promise>> { if (rowContext.getStore()) { throw new Error( 'Nested ctx.dataset() is not supported. Flatten your columns into one dataset, or keep custom per-row logic inside a single column.', ); } validatePlayAuthoringField('ctx.dataset.key', key); if (options) { validateOptionalPlayAuthoringField( 'ctx.dataset.run.description', options.description, ); validateOptionalPlayAuthoringField( 'ctx.dataset.run.onRowError', options.onRowError, ); validateOptionalPlayAuthoringField('ctx.dataset.run.mode', options.mode); validateOptionalPlayAuthoringField( 'ctx.dataset.run.undrawnColumns', options.undrawnColumns, ); if (options.key !== undefined) { validatePlayAuthoringField('ctx.dataset.run.key', options.key); } } const normalizedMapKey = this.normalizeContextKey(key, 'map'); const normalizedMapNamespace = normalizeTableNamespace(normalizedMapKey); if (this.explicitMapInvocationKeys.has(normalizedMapNamespace)) { throw new Error( `Duplicate ctx.dataset() key "${normalizedMapNamespace}" in the same play. ` + 'Each ctx.dataset() call must use a distinct idempotency key.', ); } this.explicitMapInvocationKeys.add(normalizedMapNamespace); let resolvedTableNamespace = normalizedMapNamespace; let totalInputCount = 0; let rawItems: Record[] = []; let itemsToProcess: Array> = []; let itemOriginalIndexes: number[] = []; const datasetColumnNames = Object.keys(input); const stripFieldOutputs = (row: Record) => { const stripped = cloneCsvAliasedRow(row); for (const fieldName of datasetColumnNames) { delete stripped[fieldName]; } return stripped; }; const stableInputRow = (row: Record) => stripCsvProjectedFields( Object.fromEntries( Object.entries(stripFieldOutputs(row)).filter( ([fieldName]) => !fieldName.startsWith('__deepline'), ), ), ); const userKeyOption = options?.key; let explicitKeyResolver: | ((row: Record, index: number) => string) | null = null; if (userKeyOption !== undefined) { explicitKeyResolver = (row, index) => { const inputRow = stripFieldOutputs(row); const raw = typeof userKeyOption === 'function' ? ( userKeyOption as ( row: Record, index: number, ) => string | number | readonly unknown[] )(inputRow, index) : Array.isArray(userKeyOption) ? userKeyOption.map((fieldName) => inputRow[fieldName]) : inputRow[String(userKeyOption)]; const parts = Array.isArray(raw) ? raw : [raw]; if (parts.some((part) => part === null || part === undefined)) { throw new Error( `ctx.dataset("${normalizedMapNamespace}") key returned null or undefined for row ${index}. ` + 'Return a non-empty string or number derived from a stable input column (e.g. row.email ?? row.domain).', ); } const normalizedParts = parts.map((part) => { if (typeof part === 'number') { return Number.isFinite(part) ? String(part) : ''; } return String(part).trim(); }); if (normalizedParts.some((part) => !part)) { throw new Error( `ctx.dataset("${normalizedMapNamespace}") key returned an empty value for row ${index}. ` + 'Return a non-empty string or finite number derived from a stable input column.', ); } return normalizedParts.length === 1 ? normalizedParts[0]! : JSON.stringify(normalizedParts); }; } const rowIdentity = (row: Record, index = 0) => { const runtimeKey = resolveMapRowOutcomeKey(row); if (runtimeKey) return runtimeKey; return explicitKeyResolver ? derivePlayRowIdentityFromKey( explicitKeyResolver(row, index), resolvedTableNamespace, ) : derivePlayRowIdentity(stableInputRow(row), resolvedTableNamespace); }; // Identity for SEED rows (pre-execution raw inputs), matching the server's // own derivation in startRuntimeSheetDataset: hash the FULL cleaned input // row, output-named columns INCLUDED. Two distinct input rows that differ // only in a column sharing an output's name (e.g. a CSV `domain` column + // withColumn('domain', ...)) must stay two rows — the stripped-row // `rowIdentity` fallback would collapse them to one key, silently dropping // a row from execution and the sheet. Output-insensitivity across the // execution round-trip is provided by CARRYING this key on the row // (__deeplineRowKey, which rowIdentity prefers), not by stripping. const seedRowIdentity = (row: Record, index = 0) => { const runtimeKey = resolveMapRowOutcomeKey(row); if (runtimeKey) return runtimeKey; if (explicitKeyResolver) { return derivePlayRowIdentityFromKey( explicitKeyResolver(row, index), resolvedTableNamespace, ); } return derivePlayRowIdentity( toSerializableCsvAliasedRow(row), resolvedTableNamespace, ); }; const mapMemoryLimits = resolveRuntimeMapMemoryLimits( this.#options.runtimeMapMemoryLimits, ); const materializationMemoryTracker = createRuntimeMapMaterializationTracker( { mapName: normalizedMapNamespace, budgetBytes: mapMemoryLimits.materializedBudgetBytes, }, ); const shouldStreamRuntimeBackedDataset = isPlayDataset(items) && this.#options.runtimeSheetBackedMapDatasets === true && Boolean( this.#options.onMapStart && this.#options.onMapRowsCompleted && this.#options.baseUrl && this.#options.executorToken && this.#options.playName && this.#options.runId, ); if (shouldStreamRuntimeBackedDataset) { // Net-new admission happens one source page at a time. Its progress and // returned dataset describe admitted rows, not every provider/CSV row // supplied to this run, so accumulate the admitted count below. totalInputCount = options?.mode === 'net_new' ? 0 : await items.count(); const mapScope = this.createMapExecutionScope({ logicalNamespace: normalizedMapNamespace, artifactTableNamespace: resolvedTableNamespace, explicitKey: explicitKeyResolver, }); this.setMapFrame({ mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: mapScope.artifactTableNamespace, status: 'running', totalRows: totalInputCount, completedRowKeys: [], pendingRowKeys: [], completedRowsCount: 0, pendingRowsCount: totalInputCount, failedRowsCount: 0, startedAt: Date.now(), updatedAt: Date.now(), }); this.emitExecutionEvent({ type: 'map.started', mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: mapScope.artifactTableNamespace, totalRows: totalInputCount, completedRows: 0, pendingRows: totalInputCount, at: Date.now(), }); const seenExplicitKeys = new Set(); let processedCount = 0; let successfulCount = 0; let failedCount = 0; let duplicateReuseCount = 0; let executedCount = 0; const previewRows: Record[] = []; // A bounded dataset may be materialized by authored code immediately // after run(). Keep the rows produced by this exact awaited execution so // that read-after-write does not depend on a separate sheet read becoming // visible. Datasets above the public materialize cap remain fully // streaming and sheet-backed. const materializedResultRowLimit = resolveMaterializeLimitCap(); const immediateMaterializedRows: Record[] = []; let immediateMaterializedResidentBytes = 0; let immediateMaterializedCacheEnabled = true; const disableImmediateMaterializedCache = (reason: string) => { if (!immediateMaterializedCacheEnabled) return; immediateMaterializedRows.length = 0; immediateMaterializedResidentBytes = 0; immediateMaterializedCacheEnabled = false; this.log( `Dataset ${normalizedMapNamespace} result cache disabled (${reason}); using the durable runtime sheet.`, ); }; const persistMapRows = async (rows: PersistableMapRow[]) => { if (!this.#options.onMapRowsCompleted || rows.length === 0) { return; } for (const row of rows) { assertPersistableMapRowWithinCustomerOutputLimits({ mapName: normalizedMapNamespace, row, }); } let chunk: PersistableMapRow[] = []; let chunkBytes = 0; const flushChunk = async () => { if (chunk.length === 0) return; try { const flushStartedAt = Date.now(); await this.#options.onMapRowsCompleted!({ playName: this.currentPlayName, playId: this.currentExecutionScope.logical.playId, runId: this.currentRunId, executorToken: this.#options.executorToken, tableNamespace: resolvedTableNamespace, rows: chunk, outputFields: datasetColumnNames.filter((field) => shouldPersistMapCellField(field), ), staticPipeline: this.currentStaticPipeline ?? null, }); this.resourceGovernor.observe({ sheetFlushBytes: chunkBytes, sheetFlushLatencyMs: Date.now() - flushStartedAt, }); } catch (error) { tripRuntimePersistenceLatch(this.persistenceLatch, error); throw error; } chunk = []; chunkBytes = 0; }; for (const row of rows) { const rowBytes = persistableMapRowBytes(row); if ( chunk.length >= MAP_PERSIST_CHUNK_ROWS || (chunkBytes + rowBytes > MAP_PERSIST_CHUNK_BYTES && chunk.length > 0) ) { await flushChunk(); } chunk.push(row); chunkBytes += rowBytes; } await flushChunk(); }; for await (const page of iteratePlayDatasetInputPages(items, { maxPageBytes: mapMemoryLimits.materializedBudgetBytes, estimateRowBytes: (row) => runtimeMapJsonByteLength(row) * NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER + NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES, })) { const pageRawItems = page.rows.map((item) => this.toOutputRow(item as Record), ); const pageMemoryEstimate = estimateRuntimeMapRowsMemory(pageRawItems); if ( pageMemoryEstimate.estimatedResidentBytes > mapMemoryLimits.materializedBudgetBytes ) { throw new RuntimeMapMemoryLimitError({ mapName: normalizedMapNamespace, phase: 'materialize', estimate: pageMemoryEstimate, budgetBytes: mapMemoryLimits.materializedBudgetBytes, }); } const pageStartRows = pageRawItems.map((row, index) => { const originalIndex = page.offset + index; return { ...toSerializableCsvAliasedRow(row), ...mapRowOutcomeRuntimeFields({ key: seedRowIdentity(row, originalIndex), inputIndex: originalIndex, }), }; }); const mapStartResult = await this.#options.onMapStart!( pageStartRows, resolvedTableNamespace, { playName: this.currentPlayName, playId: this.currentExecutionScope.logical.playId, runId: this.currentRunId, executorToken: this.#options.executorToken, staticPipeline: this.currentStaticPipeline, forceRefresh: this.#options.cachePolicy?.forceToolRefresh === true, inputOffset: page.offset, mode: options?.mode, }, ); resolvedTableNamespace = normalizeTableNamespace( mapStartResult.tableNamespace, ); const persistedRowIdentity = ( row: Record, index = 0, ) => resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index); const pendingRowsByKey = new Map>(); for ( let index = 0; index < mapStartResult.pendingRows.length; index += 1 ) { const row = mapStartResult.pendingRows[index]!; const rowKey = persistedRowIdentity(row, page.offset + index); if (rowKey) pendingRowsByKey.set(rowKey, row); } const completedRowsByKey = new Map>(); for (const row of mapStartResult.completedRows ?? []) { const rowKey = persistedRowIdentity(row); if (!rowKey) continue; completedRowsByKey.set(rowKey, row); } const seededItems = page.rows.map((item, index) => { const originalIndex = page.offset + index; const row = this.toOutputRow(item as Record); const rowKey = seedRowIdentity(row, originalIndex); const persisted = pendingRowsByKey.get(rowKey) ?? completedRowsByKey.get(rowKey); return { row: persisted ? cloneCsvAliasedRow( row, persistedFieldsAbsentFromInputRow(row, persisted), ) : row, index: originalIndex, }; }); const admittedSeededItems = options?.mode === 'net_new' ? seededItems.filter(({ row, index }) => pendingRowsByKey.has(seedRowIdentity(row, index)), ) : seededItems; if (options?.mode === 'net_new') { totalInputCount += admittedSeededItems.length; } const rowsToExecuteByKey = new Map< string, { row: Record; originalIndex: number } >(); for (const { row, index } of admittedSeededItems) { if (explicitKeyResolver) { const explicitKey = explicitKeyResolver(row, index); if (seenExplicitKeys.has(explicitKey)) { duplicateReuseCount += 1; processedCount += 1; continue; } seenExplicitKeys.add(explicitKey); } const rowKey = rowIdentity(row, index); if (!rowsToExecuteByKey.has(rowKey)) { rowsToExecuteByKey.set(rowKey, { row, originalIndex: index }); } else { duplicateReuseCount += 1; processedCount += 1; } } const rowsToExecuteEntries = [...rowsToExecuteByKey.entries()].map( ([rowKey, entry]) => ({ rowKey, ...entry }), ); const rowsToExecute = rowsToExecuteEntries.map((entry) => entry.row); const rowsToExecuteMemoryEstimate = estimateRuntimeMapRowsMemory(rowsToExecute); const incrementalPersistence = createRuntimeMapRowPersistence( persistMapRows, { persistCheckpoint: this.#options.onMapRowsCheckpoint ? (updates) => this.#options.onMapRowsCheckpoint!({ playName: this.currentPlayName, playId: this.currentExecutionScope.logical.playId, runId: this.currentRunId, executorToken: this.#options.executorToken, tableNamespace: resolvedTableNamespace, updates, staticPipeline: this.currentStaticPipeline ?? null, }) : undefined, onFailure: (error) => tripRuntimePersistenceLatch(this.persistenceLatch, error), }, ); this.activeMapCellMeta = new Map(); this.activeMapCheckpointUpdates = new Map(); this.setMapFrame({ mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: resolvedTableNamespace, status: 'running', totalRows: totalInputCount, completedRowKeys: [], pendingRowKeys: rowsToExecuteEntries.map((entry) => entry.rowKey), completedRowsCount: Math.max(0, processedCount - failedCount), pendingRowsCount: Math.max(0, totalInputCount - processedCount), failedRowsCount: failedCount, startedAt: this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ?? Date.now(), updatedAt: Date.now(), }); let mapResult: FieldMapRunResult; try { mapResult = await this.runFieldMap( rowsToExecute, mapScope, input as MapFieldDefinition>, options?.description, { totalRows: totalInputCount, completedRows: duplicateReuseCount, }, { emitTerminalEvent: false, onRowError: options?.onRowError, executionRowKeys: rowsToExecuteEntries.map( (entry) => entry.rowKey, ), executionRowIndexes: rowsToExecuteEntries.map( (entry) => entry.originalIndex, ), progressCompletedOffset: Math.max( 0, processedCount - failedCount, ), progressFailedOffset: failedCount, progressTotalRows: totalInputCount, largestRowBytes: rowsToExecuteMemoryEstimate.largestRowBytes, retainedRowsMemoryBudgetBytes: mapMemoryLimits.materializedBudgetBytes, activeRowsMemoryBudgetBytes: mapMemoryLimits.activeRowsBudgetBytes, incrementalPersistence, releasePersistedRowsOnMemoryPressure: true, }, ); } catch (error) { if (error instanceof FailFastMapRowsError) { const rowsToPersist = [...error.completedRows, ...error.failedRows]; await incrementalPersistence.flush(); const unpersistedRows = rowsToPersist.filter( (row) => !incrementalPersistence.isPersisted(row), ); await persistMapRows(unpersistedRows); this.activeMapCellMeta = null; this.activeMapCheckpointUpdates = null; throw error.cause; } this.activeMapCellMeta = null; this.activeMapCheckpointUpdates = null; throw error; } await incrementalPersistence.flush(); const mapCellMeta = this.activeMapCellMeta; const persistRows: PersistableMapRow[] = []; for (const row of mapResult.completedRows) { if (incrementalPersistence.isPersisted(row)) continue; const meta = mapCellMeta?.get(row.key); persistRows.push( completedMapRowOutcome({ key: row.key, data: row.data, inputIndex: row.inputIndex, cellMetaPatch: { ...(row.cellMetaPatch ?? {}), ...(meta ?? {}), }, }), ); } persistRows.push( ...mapResult.failedRows.filter( (row) => !incrementalPersistence.isPersisted(row), ), ); await persistMapRows(persistRows); this.activeMapCellMeta = null; this.activeMapCheckpointUpdates = null; for (const materializedRow of mapResult.previewRows) { if (previewRows.length < 5) previewRows.push(materializedRow); } if (!mapResult.retainedRowsComplete) { disableImmediateMaterializedCache( 'the current execution page exceeded the in-memory result budget', ); } for (const row of mapResult.completedRows) { const materializedRow = this.toMaterializedOutputRow(row.data); if ( immediateMaterializedCacheEnabled && immediateMaterializedRows.length < materializedResultRowLimit ) { const estimatedRowResidentBytes = runtimeMapJsonByteLength(materializedRow) * NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER + NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES; const observedRowCount = immediateMaterializedRows.length + 1; const observedResidentBytes = immediateMaterializedResidentBytes + estimatedRowResidentBytes; // `net_new` deliberately reports only admitted rows, so its // running total cannot predict the next page. Do not pre-scan an // unknown-count dataset just to estimate this cache. Instead, // bound the cache and active source page together while that page // remains resident. const projectedResidentBytes = options?.mode === 'net_new' ? observedResidentBytes + pageMemoryEstimate.estimatedResidentBytes : Math.ceil( (observedResidentBytes / observedRowCount) * Math.max(observedRowCount, totalInputCount - failedCount), ); if ( projectedResidentBytes > mapMemoryLimits.materializedBudgetBytes ) { // All-or-nothing cache: retaining a prefix that cannot fit once // the known input finishes gives authored code no complete fast // path. More importantly, that prefix would overlap the next // active page and can exhaust a fixed-size sandbox before the // cache reaches its own independent budget. Project from the // rows observed so far and move to the already-durable sheet // early instead of waiting for resident memory to reach a cliff. disableImmediateMaterializedCache( `projected ${projectedResidentBytes} bytes exceeds ${mapMemoryLimits.materializedBudgetBytes}-byte budget`, ); } else { immediateMaterializedRows.push(materializedRow); immediateMaterializedResidentBytes = observedResidentBytes; } } } executedCount += rowsToExecute.length; successfulCount += mapResult.completedRowCount; failedCount += mapResult.failedRowCount; processedCount += rowsToExecute.length; page.rows.length = 0; } // Isolated failures are a partial result only when at least one admitted // row completed. A fully failed admission run is systemic: preserve the // failed sheet rows for recovery, but fail the enclosing play loudly. if (totalInputCount > 0 && failedCount === totalInputCount) { const completedRows = 0; this.setMapFrame({ mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: resolvedTableNamespace, status: 'failed', totalRows: totalInputCount, completedRowKeys: [], pendingRowKeys: [], completedRowsCount: completedRows, pendingRowsCount: 0, failedRowsCount: failedCount, startedAt: this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ?? Date.now(), updatedAt: Date.now(), }); this.emitExecutionEvent({ type: 'map.failed', mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: resolvedTableNamespace, completedRows, failedRows: failedCount, totalRows: totalInputCount, at: Date.now(), }); throw new Error( `ctx.dataset("${normalizedMapNamespace}") failed for every admitted row (${failedCount}/${totalInputCount}). ` + 'Failed rows were persisted and will be retried on the next run.', ); } this.setMapFrame({ mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: resolvedTableNamespace, status: 'completed', totalRows: totalInputCount, completedRowKeys: [], pendingRowKeys: [], completedRowsCount: Math.max(0, processedCount - failedCount), pendingRowsCount: 0, failedRowsCount: failedCount, startedAt: this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ?? Date.now(), updatedAt: Date.now(), }); this.emitExecutionEvent({ type: 'map.completed', mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: resolvedTableNamespace, completedRows: Math.max(0, processedCount - failedCount), failedRows: failedCount, totalRows: totalInputCount, at: Date.now(), }); const readRuntimeBackedMapRows = async (input: { limit: number; offset: number; }) => { const result = await readRuntimeSheetDatasetRows( { baseUrl: this.#options.baseUrl!, executorToken: this.#options.executorToken!, dbSessionStrategy: this.#options.dbSessionStrategy, playName: this.#options.playName!, userEmail: this.#options.userEmail, runId: this.#options.runId!, }, { tableNamespace: resolvedTableNamespace, runId: this.#options.runId!, limit: input.limit, offset: input.offset, }, ); return result.rows.map((row) => this.toMaterializedOutputRow(row)); }; const materializeRuntimeBackedMapRows = async (limit?: number) => { if (limit !== undefined && limit <= 0) return []; const availableResultRows = Math.min( successfulCount, materializedResultRowLimit, ); if ( immediateMaterializedCacheEnabled && immediateMaterializedRows.length === availableResultRows && (limit !== undefined || successfulCount <= materializedResultRowLimit) ) { return limit === undefined ? immediateMaterializedRows.slice() : immediateMaterializedRows.slice(0, limit); } const pageSize = 1000; const materialized: Record[] = []; let offset = 0; while (true) { const remaining = limit === undefined ? pageSize : Math.max(0, limit - materialized.length); if (remaining === 0) break; const rows = await readRuntimeBackedMapRows({ limit: Math.min(pageSize, remaining), offset, }); if (rows.length === 0) break; materialized.push(...rows); if (limit !== undefined && materialized.length >= limit) { return materialized.slice(0, limit); } offset += rows.length; } return materialized; }; const materializeFullPersistedMapRows = async (limit?: number) => { if (limit !== undefined && limit <= 0) return []; const pageSize = 1000; const materialized: Record[] = []; let offset = 0; while (true) { const remaining = limit === undefined ? pageSize : Math.max(0, limit - materialized.length); if (remaining === 0) break; const rows = await readRuntimeBackedMapRows({ limit: Math.min(pageSize, remaining), offset, }); if (rows.length === 0) break; materialized.push(...rows); if (limit !== undefined && materialized.length >= limit) { return materialized.slice(0, limit); } offset += rows.length; } return materialized; }; return createDeferredPlayDataset({ datasetKind: 'map', datasetId: createRuntimeDatasetId( this.#options.playName ?? this.#options.playId ?? 'play', resolvedTableNamespace, ), count: successfulCount, backing: { storage: 'neon_sheet', sheet: { playName: this.#options.playName!, tableNamespace: resolvedTableNamespace, }, }, previewRows, residentRows: immediateMaterializedCacheEnabled && immediateMaterializedRows.length === successfulCount ? immediateMaterializedRows : null, tableNamespace: resolvedTableNamespace, workProgress: { total: totalInputCount, executed: executedCount, reused: duplicateReuseCount, skipped: duplicateReuseCount, pending: 0, failed: failedCount, ...(duplicateReuseCount > 0 ? { duplicates: { exact: duplicateReuseCount } } : {}), }, resolvers: { count: async () => successfulCount, peek: async (limit) => limit <= 0 ? [] : immediateMaterializedCacheEnabled && immediateMaterializedRows.length >= Math.min(limit, successfulCount) ? immediateMaterializedRows.slice(0, limit) : await readRuntimeBackedMapRows({ limit, offset: 0, }), materialize: materializeRuntimeBackedMapRows, materializeFullPersistedDataset: materializeFullPersistedMapRows, iterate: () => ({ async *[Symbol.asyncIterator]() { if ( immediateMaterializedCacheEnabled && immediateMaterializedRows.length === successfulCount ) { for (const row of immediateMaterializedRows) yield row; return; } const pageSize = 1000; let offset = 0; while (true) { const rows = await readRuntimeBackedMapRows({ limit: pageSize, offset, }); if (rows.length === 0) return; for (const row of rows) { yield row; } offset += rows.length; } }, }) as AsyncIterable>, }, }); } const rawMaterializedItems = await materializePlayDatasetInput(items, { onRow: (row) => { materializationMemoryTracker.track(row); }, }); // Silent dedupe of duplicate explicit map keys: keep the first row per // canonical key and drop subsequent duplicates. Deduping the materialized // input here keeps every downstream derivation (rawItems, itemsToProcess, // counts, sheet writes, resumable keys) consistent — the resolver derives // keys from row content, so dropped rows are excluded everywhere with no // index mismatch. Replaces the prior throw-on-duplicate behavior. const dedupedMaterialized = explicitKeyResolver ? dedupeExplicitMapKeyRows({ rows: rawMaterializedItems, resolver: (item, index) => explicitKeyResolver( this.toOutputRow(item as Record), index, ), }) : { rows: rawMaterializedItems, droppedCount: 0, duplicateKeys: [] }; if (dedupedMaterialized.droppedCount > 0) { const keySample = dedupedMaterialized.duplicateKeys.join(', '); this.log( `deduped ${dedupedMaterialized.droppedCount} duplicate dataset key(s) for ctx.dataset("${normalizedMapNamespace}"); keeping first occurrence` + (keySample ? ` (e.g. ${keySample})` : ''), ); } const materializedItems = dedupedMaterialized.rows; totalInputCount = materializedItems.length; rawItems = materializedItems.map((item) => this.toOutputRow(item)); itemsToProcess = materializedItems.map((item) => this.toOutputRow(item as Record), ); itemOriginalIndexes = materializedItems.map((_item, index) => index); const mapStartSeedRowsByKey = new Map>(); if (this.#options.onMapStart) { // ALWAYS stamp the canonical seed row key — including no-explicit-key // datasets — using seedRowIdentity (full-input-row hash, matching the // server's own derivation). Without the stamp, seed and completion keys // diverge whenever an input column shares a name with an output column // (e.g. a CSV `domain` column + withColumn('domain', ...)): the seed // row persists under the full-row key while completion/visibility used // a stripped-row key, so the input-index repair flipped the seed row // terminal under the OLD key and the visibility barrier polled a key // that never existed — "Runtime sheet visibility mismatch ... saw 0; // write reported 1" (28-packaged-literal-csv on the Daytona runner). // Stamping a STRIPPED-row key instead would collapse distinct input // rows differing only in output-named columns into one sheet row // (resultView expected 2, got 1). Full-input identity keeps them // distinct; the carried __deeplineRowKey keeps identity stable across // the execution round-trip. // // toSerializableCsvAliasedRow (not a plain spread): projected CSV // aliases ride on non-enumerable props after stripCsvProjectionMetadata, // so a spread would silently drop them before the sheet write and the // aliases would never persist as visible input cells. const mapStartRows = rawItems.map((row, index) => ({ ...toSerializableCsvAliasedRow(row), ...mapRowOutcomeRuntimeFields({ key: seedRowIdentity(row, index), }), })); const mapStartResult = await this.#options.onMapStart( mapStartRows, resolvedTableNamespace, { playName: this.currentPlayName, playId: this.currentExecutionScope.logical.playId, runId: this.currentRunId, executorToken: this.#options.executorToken, staticPipeline: this.currentStaticPipeline, forceRefresh: this.#options.cachePolicy?.forceToolRefresh === true, mode: options?.mode, }, ); resolvedTableNamespace = normalizeTableNamespace( mapStartResult.tableNamespace, ); const persistedRowIdentity = (row: Record, index = 0) => resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index); const pendingRowsByKey = new Map>(); for ( let index = 0; index < mapStartResult.pendingRows.length; index += 1 ) { const row = mapStartResult.pendingRows[index]!; const rowKey = persistedRowIdentity(row, index); if (rowKey) pendingRowsByKey.set(rowKey, row); } for (const row of mapStartResult.completedRows ?? []) { const rowKey = persistedRowIdentity(row); if (!rowKey) continue; mapStartSeedRowsByKey.set(rowKey, row); } const seededItems = materializedItems .map((item, index) => ({ row: this.toOutputRow(item as Record), index, })) .map(({ row, index }) => { // Look up by the SAME seed identity the stamp above used — the // merged persisted row then carries __deeplineRowKey, which every // downstream rowIdentity call prefers over re-derivation. const rowKey = seedRowIdentity(row, index); const persisted = pendingRowsByKey.get(rowKey) ?? mapStartSeedRowsByKey.get(rowKey); // Restore persisted OUTPUT columns, cell meta, and the carried // __deeplineRowKey so key functions and column resolvers keep seeing // projected values/aliases across the sheet round-trip. But the // CURRENT run's INPUT columns are authoritative and must win: a // persisted row is matched only by the map key, so for a play whose // key does not capture every input column it can be a DIFFERENT run's // row that merely shares that key. Letting its stale input columns // overwrite this run's inputs corrupts input-derived tool arguments // (e.g. scenario 87 vs 78 collide on key `id`='target' but pass // distinct `key` columns to the probe tool, flipping a terminal // failure into a spurious success). Merge only persisted fields // absent from the fresh input row. return { row: persisted ? cloneCsvAliasedRow( row, persistedFieldsAbsentFromInputRow(row, persisted), ) : row, index, }; }); const admittedItems = options?.mode === 'net_new' ? seededItems.filter(({ row, index }) => pendingRowsByKey.has(seedRowIdentity(row, index)), ) : seededItems; rawItems = admittedItems.map((item) => item.row); itemsToProcess = admittedItems.map((item) => item.row); itemOriginalIndexes = admittedItems.map((item) => item.index); totalInputCount = admittedItems.length; } const mapScope = this.createMapExecutionScope({ logicalNamespace: normalizedMapNamespace, artifactTableNamespace: resolvedTableNamespace, explicitKey: explicitKeyResolver, }); const completedRowKeys: string[] = []; const pendingRowKeys = itemsToProcess.map((item, index) => rowIdentity(this.toOutputRow(item), itemOriginalIndexes[index] ?? index), ); this.setMapFrame({ mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: mapScope.artifactTableNamespace, status: 'running', totalRows: totalInputCount, completedRowKeys, pendingRowKeys, startedAt: Date.now(), updatedAt: Date.now(), }); this.emitExecutionEvent({ type: 'map.started', mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: mapScope.artifactTableNamespace, totalRows: totalInputCount, completedRows: completedRowKeys.length, pendingRows: pendingRowKeys.length, at: Date.now(), }); const rowsToExecuteByKey = new Map< string, { row: Record; originalIndex: number } >(); for (let index = 0; index < itemsToProcess.length; index += 1) { const row = itemsToProcess[index]!; const originalIndex = itemOriginalIndexes[index] ?? index; const rowKey = rowIdentity(row, originalIndex); if (!rowsToExecuteByKey.has(rowKey)) { rowsToExecuteByKey.set(rowKey, { row, originalIndex }); } } const rowsToExecuteEntries = [...rowsToExecuteByKey.entries()].map( ([rowKey, entry]) => ({ rowKey, ...entry }), ); const rowsToExecute = rowsToExecuteEntries.map((entry) => entry.row); const rowsToExecuteMemoryEstimate = estimateRuntimeMapRowsMemory(rowsToExecute); const duplicateReuseCount = Math.max( 0, itemsToProcess.length - rowsToExecute.length, ); this.activeMapCellMeta = new Map(); this.activeMapCheckpointUpdates = new Map(); const staleCompletionKeys = new Set(); const persistMapRows = async (rows: PersistableMapRow[]) => { if (!this.#options.onMapRowsCompleted || rows.length === 0) { return; } for (const row of rows) { assertPersistableMapRowWithinCustomerOutputLimits({ mapName: normalizedMapNamespace, row, }); } let chunk: PersistableMapRow[] = []; let chunkBytes = 0; const flushChunk = async () => { if (chunk.length === 0) return; try { const flushStartedAt = Date.now(); const writeResult = await this.#options.onMapRowsCompleted!({ playName: this.currentPlayName, playId: this.currentExecutionScope.logical.playId, runId: this.currentRunId, executorToken: this.#options.executorToken, tableNamespace: resolvedTableNamespace, rows: chunk, outputFields: datasetColumnNames.filter((field) => shouldPersistMapCellField(field), ), staticPipeline: this.currentStaticPipeline ?? null, }); if (writeResult) { for (const key of writeResult.staleDroppedKeys ?? []) { staleCompletionKeys.add(key); } } this.resourceGovernor.observe({ sheetFlushBytes: chunkBytes, sheetFlushLatencyMs: Date.now() - flushStartedAt, }); } catch (error) { // Output-sheet flush failed: trip the breaker so the dispatch loops // stop dispatching new provider calls for the rest of this map. tripRuntimePersistenceLatch(this.persistenceLatch, error); throw error; } chunk = []; chunkBytes = 0; }; for (const row of rows) { const rowBytes = JSON.stringify(row).length; if ( chunk.length >= MAP_PERSIST_CHUNK_ROWS || (chunkBytes + rowBytes > MAP_PERSIST_CHUNK_BYTES && chunk.length > 0) ) { await flushChunk(); } chunk.push(row); chunkBytes += rowBytes; } await flushChunk(); }; const incrementalPersistence = this.#options.onMapRowsCompleted ? createRuntimeMapRowPersistence(persistMapRows, { persistCheckpoint: this.#options.onMapRowsCheckpoint ? (updates) => this.#options.onMapRowsCheckpoint!({ playName: this.currentPlayName, playId: this.currentExecutionScope.logical.playId, runId: this.currentRunId, executorToken: this.#options.executorToken, tableNamespace: resolvedTableNamespace, updates, staticPipeline: this.currentStaticPipeline ?? null, }) : undefined, onFailure: (error) => tripRuntimePersistenceLatch(this.persistenceLatch, error), }) : null; let mapResult: FieldMapRunResult; try { mapResult = await this.runFieldMap( rowsToExecute, mapScope, input as MapFieldDefinition>, options?.description, { totalRows: totalInputCount, completedRows: duplicateReuseCount, }, { emitTerminalEvent: false, onRowError: options?.onRowError, executionRowKeys: rowsToExecuteEntries.map((entry) => entry.rowKey), executionRowIndexes: rowsToExecuteEntries.map( (entry) => entry.originalIndex, ), largestRowBytes: rowsToExecuteMemoryEstimate.largestRowBytes, retainedRowsMemoryBudgetBytes: mapMemoryLimits.materializedBudgetBytes, activeRowsMemoryBudgetBytes: mapMemoryLimits.activeRowsBudgetBytes, incrementalPersistence, }, ); } catch (error) { if (error instanceof FailFastMapRowsError) { const rowsToPersist = [...error.completedRows, ...error.failedRows]; await incrementalPersistence?.flush(); const unpersistedRows = incrementalPersistence ? rowsToPersist.filter( (row) => !incrementalPersistence.isPersisted(row), ) : rowsToPersist; const persistStartedAt = Date.now(); await persistMapRows(unpersistedRows); if (unpersistedRows.length > 0) { this.log( `Persisted ${unpersistedRows.length} fail-fast rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`, ); } this.activeMapCellMeta = null; this.activeMapCheckpointUpdates = null; throw error.cause; } this.activeMapCellMeta = null; this.activeMapCheckpointUpdates = null; throw error; } const resultsByKey = new Map>(); for (const row of mapResult.completedRows) { if (row.key) resultsByKey.set(row.key, row.data); } const failedRowKeys = new Set( mapResult.failedRows.map((row) => row.key).filter(Boolean), ); const directCompletedResults = mapResult.completedRows.map((row) => this.toPublicOutputRow(row.data), ); let results = mapResult.failedRows.length === 0 && directCompletedResults.length === rawItems.length ? directCompletedResults : rawItems.flatMap((rawItem, index) => { const rowKey = rowIdentity(rawItem, index); if (rowKey && failedRowKeys.has(rowKey)) { return []; } return [ this.toPublicOutputRow(resultsByKey.get(rowKey) ?? rawItem), ]; }); const executedCount = rowsToExecute.length; const reusedCount = duplicateReuseCount; // Persist executed rows to the tenant runtime sheet — the sheet is the // source of truth, not this in-memory results array. Chunked by rows AND // bytes so large cells (scraped pages) never produce oversized writes. if (resultsByKey.size > 0 || mapResult.failedRows.length > 0) { await incrementalPersistence?.flush(); const mapCellMeta = this.activeMapCellMeta; const persistRows: PersistableMapRow[] = []; for (const row of mapResult.completedRows) { if (incrementalPersistence?.isPersisted(row)) continue; const rowKey = row.key; const meta = mapCellMeta?.get(rowKey); persistRows.push( completedMapRowOutcome({ key: rowKey, data: row.data, cellMetaPatch: { ...(row.cellMetaPatch ?? {}), ...(meta ?? {}), }, }), ); } persistRows.push( ...mapResult.failedRows.filter( (row) => !incrementalPersistence?.isPersisted(row), ), ); const persistStartedAt = Date.now(); await persistMapRows(persistRows); if (persistRows.length > 0) { this.log( `Persisted ${persistRows.length} executed rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`, ); } } this.activeMapCellMeta = null; this.activeMapCheckpointUpdates = null; if (staleCompletionKeys.size > 0) { results = results.filter((row, index) => { const key = rowIdentity(row, itemOriginalIndexes[index] ?? index); return !staleCompletionKeys.has(key); }); } const durableCompletedRows = Math.max( 0, reusedCount + mapResult.completedRows.length - staleCompletionKeys.size, ); const durableFailedRows = mapResult.failedRows.length; const terminalFrame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId]; if (terminalFrame) { this.setMapFrame({ ...terminalFrame, status: 'completed', // Incremental rows add keys after their commit. Pure maps use the // durable aggregate count below instead of materializing up to millions // of keys only to mark the terminal frame. completedRowKeys: terminalFrame.completedRowKeys, pendingRowKeys: [], completedRowsCount: durableCompletedRows, pendingRowsCount: 0, failedRowsCount: durableFailedRows, activeBoundaryId: null, updatedAt: Date.now(), }); } this.emitExecutionEvent({ type: 'map.completed', mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: resolvedTableNamespace, completedRows: durableCompletedRows, failedRows: durableFailedRows, totalRows: totalInputCount, ...this.inlineChildAggregateEventFields(), at: Date.now(), }); if (this.#options.onMapRowsCompleted && staleCompletionKeys.size === 0) { results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({ mapName: normalizedMapNamespace, tableNamespace: resolvedTableNamespace, runId: this.#options.runId, expectedRows: totalInputCount, currentRows: results, failedRowCount: mapResult.failedRows.length, log: (line) => this.log(line), readPersistedRows: async (readInput) => { if ( !this.#options.baseUrl || !this.#options.executorToken || !this.#options.playName || !this.#options.runId ) { throw new Error( `Runtime sheet finalization mismatch for ctx.dataset("${normalizedMapNamespace}"): ` + 'cannot verify terminal persisted rows because the Node runtime context is missing ' + 'baseUrl, executorToken, playName, or runId.', ); } const result = await readRuntimeSheetDatasetRows( { baseUrl: this.#options.baseUrl, executorToken: this.#options.executorToken, playName: this.#options.playName, userEmail: this.#options.userEmail, runId: this.#options.runId, }, { tableNamespace: resolvedTableNamespace, runId: this.#options.runId, rowMode: readInput.rowMode, limit: readInput.limit, offset: readInput.offset, }, ); return result.rows; }, }); } // This map's terminal rows have already crossed the persistence barrier // above. Keep that verified snapshot as the authored return value instead // of immediately re-reading the Runtime Sheet projection. A second read // can observe a lagging/dynamically incomplete physical projection even // though the terminal write succeeded (notably for inline-child object // outputs on Absurd), making `materialize()` disagree with durable truth. // Neon remains the backing for later API/export reads of this handle. const terminalRows = results.map((row) => this.toMaterializedOutputRow(row), ); const materializeTerminalRows = async (limit?: number) => limit === undefined ? terminalRows.slice() : terminalRows.slice(0, Math.max(0, limit)); const runtimeSheetBacked = this.#options.runtimeSheetBackedMapDatasets === true && Boolean( this.#options.baseUrl && this.#options.executorToken && this.#options.playName && this.#options.runId, ); const materializeFullPersistedMapRows = async (limit?: number) => { if (!runtimeSheetBacked) { throw new Error( 'The full persisted dataset is unavailable because this run result has no Runtime Sheet backing.', ); } if (limit !== undefined && limit <= 0) return []; const pageSize = 1000; const materialized: Record[] = []; let offset = 0; while (true) { const remaining = limit === undefined ? pageSize : Math.max(0, limit - materialized.length); if (remaining === 0) break; const result = await readRuntimeSheetDatasetRows( { baseUrl: this.#options.baseUrl!, executorToken: this.#options.executorToken!, dbSessionStrategy: this.#options.dbSessionStrategy, playName: this.#options.playName!, userEmail: this.#options.userEmail, runId: this.#options.runId!, }, { tableNamespace: resolvedTableNamespace, runId: this.#options.runId!, limit: Math.min(pageSize, remaining), offset, }, ); const rows = result.rows.map((row) => this.toMaterializedOutputRow(row), ); if (rows.length === 0) break; materialized.push(...rows); if (limit !== undefined && materialized.length >= limit) { return materialized.slice(0, limit); } offset += rows.length; } return materialized; }; return createDeferredPlayDataset({ datasetKind: 'map', datasetId: createRuntimeDatasetId( this.#options.playName ?? this.#options.playId ?? 'play', resolvedTableNamespace, ), count: results.length, backing: runtimeSheetBacked ? { storage: 'neon_sheet', sheet: { playName: this.#options.playName!, tableNamespace: resolvedTableNamespace, }, } : undefined, previewRows: results .slice(0, 5) .map((row) => this.toMaterializedOutputRow(row)), residentRows: terminalRows, tableNamespace: resolvedTableNamespace, workProgress: { total: totalInputCount, executed: executedCount, reused: reusedCount, skipped: reusedCount, pending: 0, failed: mapResult.failedRows.length, ...(duplicateReuseCount > 0 ? { duplicates: { exact: duplicateReuseCount } } : {}), }, resolvers: { count: async () => results.length, peek: async (limit) => terminalRows.slice(0, Math.max(0, limit)), materialize: materializeTerminalRows, ...(runtimeSheetBacked ? { materializeFullPersistedDataset: materializeFullPersistedMapRows, } : {}), iterate: () => ({ async *[Symbol.asyncIterator]() { for (const row of terminalRows) { yield row; } }, }) as AsyncIterable>, }, }); } private async runFieldMap>( items: T[], mapScope: MapExecutionScope, definition: MapFieldDefinition, description?: string, executionSummary?: { totalRows: number; completedRows: number; }, runtimeOptions?: { emitTerminalEvent?: boolean; onRowError?: 'isolate' | 'fail'; executionRowKeys?: string[]; executionRowIndexes?: number[]; /** * Opt-in cap on how many row resolvers run their body concurrently. When * unset, the Governor's rowDefault is used. Excess rows stay pending as * indexes until a worker is free. */ concurrency?: number; largestRowBytes?: number; retainedRowsMemoryBudgetBytes?: number; activeRowsMemoryBudgetBytes?: number; progressCompletedOffset?: number; progressFailedOffset?: number; progressTotalRows?: number; incrementalPersistence?: RuntimeMapRowPersistence | null; /** * Runtime Sheet-backed maps may discard their optional in-memory result * cache once it reaches the resident-byte budget. The durable sheet is * already authoritative; retaining the same completed payloads until a * source page ends must never turn a successful large map into an OOM. */ releasePersistedRowsOnMemoryPressure?: boolean; }, ): Promise { const fieldEntries = Object.entries(definition); const datasetColumnNames = fieldEntries.map(([fieldName]) => fieldName); /** Columns this map writes; excluded from a row's input read order. */ const datasetColumnSet = new Set(datasetColumnNames); const visibleFields = fieldEntries .map(([fieldName]) => fieldName) .filter((fieldName) => shouldPersistMapCellField(fieldName)); const normalizedTableNamespace = mapScope.artifactTableNamespace; const rowIdentity = (row: Record, index = 0) => resolveMapRowOutcomeKey(row) ?? mapScope.rowIdentity( stripCsvProjectedFields( Object.fromEntries( Object.entries(row).filter( ([fieldName]) => !datasetColumnNames.includes(fieldName) && !fieldName.startsWith('__deepline'), ), ), ), index, ); const executionRowKey = (row: Record, index: number) => runtimeOptions?.executionRowKeys?.[index] ?? rowIdentity(row, index); const executionRowIndex = (index: number) => runtimeOptions?.executionRowIndexes?.[index] ?? index; const totalRows = Math.max( runtimeOptions?.progressTotalRows ?? executionSummary?.totalRows ?? items.length, items.length, ); const completedRows = Math.min( executionSummary?.completedRows ?? 0, totalRows, ); const pendingRows = Math.max(0, items.length); const emitTerminalEvent = runtimeOptions?.emitTerminalEvent !== false; const failFastOnRowError = runtimeOptions?.onRowError === 'fail'; const completedRowsToPersist: PersistableMapRow[] = []; const failedRowsToPersist: PersistableMapRow[] = []; const previewRows: Record[] = []; let completedRowCount = 0; let failedRowCount = 0; let retainedRowsComplete = true; const incrementalPersistence = runtimeOptions?.incrementalPersistence ?? null; let mapStallObserver: RuntimeMapStallObserver | null = null; const retainedRowsMemoryTracker = createRuntimeMapRetainedRowsTracker({ mapName: normalizedTableNamespace, budgetBytes: runtimeOptions?.retainedRowsMemoryBudgetBytes, }); const retainPersistedRow = ( row: PersistableMapRow, target: PersistableMapRow[], ): void => { if (!retainedRowsComplete) return; try { retainedRowsMemoryTracker.track(row); target.push(row); } catch (error) { if ( !runtimeOptions?.releasePersistedRowsOnMemoryPressure || !incrementalPersistence || !(error instanceof RuntimeMapMemoryLimitError) ) { throw error; } completedRowsToPersist.length = 0; failedRowsToPersist.length = 0; retainedRowsComplete = false; this.log( `Dataset ${normalizedTableNamespace} released its in-memory result cache after ${completedRowCount + failedRowCount} row(s); using the durable Runtime Sheet.`, ); } }; const enqueueIncrementalPersist = ( row: PersistableMapRow, onCommitted: () => void, ): Promise => { if (!incrementalPersistence) { onCommitted(); return Promise.resolve(); } const settlement = incrementalPersistence.persistRows([row]); const stall = mapStallObserver?.terminalQueued(); void settlement.committed .then(() => { stall?.committed(); onCommitted(); }) .catch(() => { stall?.failed(); // The terminal map barrier awaits the writer and surfaces the same // latched failure. This continuation only publishes durable progress. }); return settlement.admitted.then( () => stall?.admitted(), (error) => { stall?.failed(); throw error; }, ); }; if (completedRows > 0 || pendingRows !== totalRows) { this.log( `Starting map over ${totalRows} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace}; ${completedRows} duplicate keys skipped; ${pendingRows} pending)`, ); } else { this.log( `Starting map over ${items.length} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace})`, ); } this.processedRowCount = items.length; const datasetStep: Extract = { type: 'dataset', items: items.length, fields: visibleFields, substeps: [], description: normalizeStepDescription(description), }; this.steps.push(datasetStep); this.activeDatasetStep = datasetStep; // Live row-key sets for this map invocation, mutated in place (O(1) per // row). The previous implementation rebuilt both Sets from the checkpoint // arrays, re-spread them into fresh arrays, AND deep-cloned the frame on // every row — O(rows²) copies that collapsed 150k-row maps to ~30 rows/s. // The checkpoint frame is now materialized on a throttle (every // MAP_FRAME_FLUSH_ROW_INTERVAL rows / MAP_FRAME_FLUSH_INTERVAL_MS / any // status, boundary, or explicit-event transition), which only narrows how // fresh the replay frame is between persisted checkpoints — re-executed // rows remain idempotent via deterministic keys and reuse detection. let liveFrameSets: { completedRowKeys: Set; pendingRowKeys: Set; } | null = null; let lastFrameFlushAt = 0; let rowsSinceFrameFlush = 0; const updateMapFrameProgress = (input: { status?: MapExecutionFrame['status']; completedRowKey?: string | null; pendingRowKey?: string | null; completedRowKeys?: string[]; activeBoundaryId?: string | null; failedDelta?: number; failedRowKey?: string | null; emitEventType?: PlayExecutionEvent['type']; }) => { const existing = this.checkpoint.mapFrames?.[mapScope.mapInvocationId] ?? null; if (!existing) { return; } liveFrameSets ??= { completedRowKeys: new Set(existing.completedRowKeys), pendingRowKeys: new Set(existing.pendingRowKeys), }; const { completedRowKeys, pendingRowKeys } = liveFrameSets; const completedBefore = completedRowKeys.size; if (input.completedRowKey?.trim()) { const completedRowKey = input.completedRowKey.trim(); completedRowKeys.add(completedRowKey); pendingRowKeys.delete(completedRowKey); } if (input.completedRowKeys) { for (const rawKey of input.completedRowKeys) { const completedRowKey = rawKey.trim(); if (!completedRowKey) continue; completedRowKeys.add(completedRowKey); pendingRowKeys.delete(completedRowKey); } } if (input.pendingRowKey?.trim()) { pendingRowKeys.add(input.pendingRowKey.trim()); } if (input.failedRowKey?.trim()) { pendingRowKeys.delete(input.failedRowKey.trim()); } rowsSinceFrameFlush += completedRowKeys.size - completedBefore; const isTransition = input.status !== undefined || input.activeBoundaryId !== undefined || input.emitEventType !== undefined; const now = Date.now(); // Small maps keep per-row progress (live UX, negligible cost); the // row/time throttle only kicks in where the per-row materialization // cost matters. const shouldFlush = isTransition || totalRows <= MAP_FRAME_FLUSH_ROW_INTERVAL || rowsSinceFrameFlush >= MAP_FRAME_FLUSH_ROW_INTERVAL || now - lastFrameFlushAt >= MAP_FRAME_FLUSH_INTERVAL_MS; if (shouldFlush) { lastFrameFlushAt = now; rowsSinceFrameFlush = 0; const nextFrame: MapExecutionFrame = { ...existing, status: input.status ?? existing.status, completedRowKeys: [...completedRowKeys], pendingRowKeys: [...pendingRowKeys], ...(input.activeBoundaryId !== undefined ? { activeBoundaryId: input.activeBoundaryId } : {}), updatedAt: now, }; this.setMapFrame(nextFrame); } const progressEventType = input.emitEventType ?? (shouldFlush && completedRowKeys.size > completedBefore ? 'map.progress' : null); if (progressEventType) { const failedRows = Math.max( 0, (runtimeOptions?.progressFailedOffset ?? 0) + (items.length - completedRowKeys.size - pendingRowKeys.size), ); this.emitExecutionEvent({ type: progressEventType, mapInvocationId: mapScope.mapInvocationId, mapNodeId: mapScope.mapNodeId ?? null, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: mapScope.artifactTableNamespace, completedRows: (runtimeOptions?.progressCompletedOffset ?? 0) + completedRowKeys.size, failedRows, totalRows, // Inline child aggregates ride the single-writer progress event so // fan-out never contends on a per-child mutation. See ADR 0013. ...this.inlineChildAggregateEventFields(), at: Date.now(), } as PlayExecutionEvent); } }; if (this.canUsePureJsMapFastPath(definition)) { const results = await this.runPureFieldMap( items, fieldEntries, visibleFields, normalizedTableNamespace, executionRowKey, ); const pureResultsMemoryEstimate = estimateRuntimeMapRowsMemory(results); const retainedRowsBudgetBytes = runtimeOptions?.retainedRowsMemoryBudgetBytes ?? resolveRuntimeMapMemoryLimits(this.#options.runtimeMapMemoryLimits) .materializedBudgetBytes; if ( pureResultsMemoryEstimate.estimatedResidentBytes > retainedRowsBudgetBytes ) { throw new RuntimeMapMemoryLimitError({ mapName: normalizedTableNamespace, phase: 'retained_rows', estimate: pureResultsMemoryEstimate, budgetBytes: retainedRowsBudgetBytes, }); } // One batched frame update for the whole pure map. The per-row loop // this replaces ran AFTER every row had already computed, emitting 150k // post-hoc map.progress events and re-copying the completed-keys array // per row — all theater, no live progress value. if (!incrementalPersistence) { updateMapFrameProgress({ completedRowKeys: results.map((_row, index) => executionRowKey( this.toOutputRow(items[index] as Record), index, ), ), }); } if (emitTerminalEvent) { updateMapFrameProgress({ status: 'completed', activeBoundaryId: null, emitEventType: 'map.completed', }); } this.lastDatasetStep = datasetStep; this.activeDatasetStep = null; this.log( `Map completed: ${results.length + completedRows} results (${results.length} succeeded, 0 failed, ${completedRows} duplicate keys skipped)`, ); return { completedRows: results.map((row, index) => completedMapRowOutcome({ key: executionRowKey( this.toOutputRow(items[index] as Record), index, ), inputIndex: executionRowIndex(index), data: this.toPersistedOutputRow(row), }), ), failedRows: [], completedRowCount: results.length, failedRowCount: 0, previewRows: results.slice(0, 5), retainedRowsComplete: true, }; } this.initializeRowStates(items); // Bound on how many row resolver bodies exist at once. An explicit // `concurrency` is clamped to [1, policy.concurrency.rowMax]; unset uses // policy.concurrency.rowDefault (resolved by the Governor). Pending rows are // indexes, not promises, so they do not pre-claim receipts, queue tools, or // retain row closures before admission. Pure-JS maps use a separate fast // path and never reach here. const requestedRowConcurrency = this.resourceGovernor.resolveRowConcurrency( runtimeOptions?.concurrency, ); const largestRowBytes = runtimeOptions?.largestRowBytes ?? estimateRuntimeMapRowsMemory(items).largestRowBytes; const rowAdmission = resolveRuntimeMapRowAdmission({ rowCount: items.length, requestedConcurrency: requestedRowConcurrency, largestRowBytes, activeRowsBudgetBytes: runtimeOptions?.activeRowsMemoryBudgetBytes, }); const rowConcurrencyLimit = rowAdmission.concurrency; if (items.length > 0 && rowConcurrencyLimit === 0) { throw new RuntimeMapMemoryLimitError({ mapName: normalizedTableNamespace, phase: 'active_rows', estimate: { rowCount: items.length, serializedBytes: largestRowBytes, largestRowBytes, estimatedResidentBytes: rowAdmission.estimatedBytesPerActiveRow, }, budgetBytes: runtimeOptions?.activeRowsMemoryBudgetBytes ?? resolveRuntimeMapMemoryLimits().activeRowsBudgetBytes, activeRows: 1, rowConcurrency: requestedRowConcurrency, }); } this.resourceGovernor.observe({ memoryBytes: rowAdmission.estimatedResidentBytes, }); mapStallObserver = new RuntimeMapStallObserver({ getWriterDiagnostics: () => incrementalPersistence?.diagnostics() ?? null, log: (line) => this.log(line), mapName: normalizedTableNamespace, pageOffset: runtimeOptions?.executionRowIndexes?.[0] ?? runtimeOptions?.progressCompletedOffset ?? 0, intervalMs: this.#options.runtimeMapStallLogIntervalMs, }); const settledResults = new Array( items.length, ); let nextRowIndex = 0; let firstRowError: unknown = null; const runRow = async (idx: number): Promise => { const item = items[idx]; const baseRow = this.toOutputRow(item); const rowKey = executionRowKey(baseRow, idx); const rowIndex = executionRowIndex(idx); const computedFields: Record = {}; let activeFieldName: string | null = null; // ADR 0019: the row's read order, computed once. Every cell reads a // prefix of it — the inputs, then each column this run wrote — so this // loop appends rather than rebuilding a per-cell set (which was // O(columns squared) in both time and allocations). if (this.docflowCaptureEnabled) { this.recordRowReadColumns( rowKey, this.rowInputReadColumns(baseRow, datasetColumnSet), ); } if (this.docflowCaptureEnabled) { this.seedActiveCellProducersFromDurableRow(rowKey, baseRow); } // Global row slot keeps concurrent maps in the same run under rowMax. The // worker pool below enforces this map's requested/default concurrency. const globalRowSlot = await this.resourceGovernor.acquireRow({ estimatedBytes: runtimeMapJsonByteLength(baseRow), }); const resolverObservation = mapStallObserver?.resolverStarted(); let resolverObservationActive = resolverObservation !== undefined; const finishResolver = () => { if (!resolverObservationActive) return; resolverObservationActive = false; resolverObservation?.finish(); }; try { for (const [fieldName, resolver] of fieldEntries) { activeFieldName = fieldName; // Recorded before the first patch for the cell, so every later patch // (tool legs, completion) re-states the row order for free. this.recordRowReadCell( rowKey, fieldName, shouldPersistMapCellField(fieldName), ); this.emitScopedFieldMetaUpdate({ rowId: idx, key: rowKey, tableNamespace: normalizedTableNamespace, fieldName, status: 'running', rowStatus: 'running', stage: fieldName, dataPatch: {}, }); let value: unknown; let cellValue: unknown; try { value = await rowContext.run( { rowId: idx, fieldName, tableNamespace: normalizedTableNamespace, rowKey, mapScope, mapStallObserver: mapStallObserver ?? undefined, mapStallResolverToken: resolverObservation?.token, }, async () => await this.resolveMapFieldValue( resolver, item, this.rehydrateRowFields( cloneCsvAliasedRow(baseRow, computedFields), ), rowIndex, this.previousCellForField(baseRow, fieldName), ), ); cellValue = await this.serializeCellValue(value); } catch (error) { if ( isPlayRowExecutionSuspendedError(error) || error instanceof PlayExecutionSuspendedError ) { throw error; } if (isRowIsolationExemptError(error)) { throw error; } value = null; computedFields[fieldName] = value; const formattedError = this.formatRuntimeError(error); this.emitScopedFieldMetaUpdate({ rowId: idx, key: rowKey, tableNamespace: normalizedTableNamespace, fieldName, status: 'failed', rowStatus: 'running', stage: 'failed', provider: null, error: formattedError, dataPatch: shouldPersistMapCellField(fieldName) ? { [fieldName]: value } : {}, }); if (failFastOnRowError) { const failedRow: PersistableMapRow = failedMapRowOutcome({ key: rowKey, inputIndex: rowIndex, data: this.toPersistedOutputRow( cloneCsvAliasedRow(baseRow, computedFields), ), ...(this.activeMapCellMeta?.get(rowKey) ? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) } : {}), error: formattedError, }); // This row has not entered incremental persistence because the // fail-fast error is about to unwind the resolver. Retain this // one terminal fact even if the optional completed-row cache was // already released, so the outer failure barrier can persist it. if (retainedRowsComplete) { retainPersistedRow(failedRow, failedRowsToPersist); } if (!retainedRowsComplete) { failedRowsToPersist.push(failedRow); } throw error; } const failedData = this.toPersistedOutputRow( cloneCsvAliasedRow(baseRow, computedFields), ); const cellMetaPatch = this.activeMapCellMeta?.get(rowKey); const failedRow: PersistableMapRow = failedMapRowOutcome({ key: rowKey, inputIndex: rowIndex, data: failedData, ...(cellMetaPatch ? { cellMetaPatch } : {}), error: formattedError, }); failedRowCount += 1; retainPersistedRow(failedRow, failedRowsToPersist); this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, { rowId: idx, status: 'failed', stage: 'failed', provider: null, error: formattedError, dataPatch: {}, }); finishResolver(); await enqueueIncrementalPersist(failedRow, () => { this.clearActiveMapCheckpointUpdate( rowKey, normalizedTableNamespace, ); updateMapFrameProgress({ failedRowKey: rowKey }); }); return FAILED_ROW; } computedFields[fieldName] = cellValue; const currentCellMeta = this.activeMapCellMeta?.get(rowKey)?.[fieldName]; const currentCellRecord = currentCellMeta && typeof currentCellMeta === 'object' && !Array.isArray(currentCellMeta) ? (currentCellMeta as { status?: unknown; reused?: unknown }) : null; const currentCellStatus = currentCellRecord?.status ?? null; if (currentCellStatus === 'skipped') { if ( shouldPersistMapCellField(fieldName) && Object.prototype.hasOwnProperty.call(computedFields, fieldName) ) { this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, { rowId: idx, status: 'running', stage: 'skipped', provider: null, error: null, dataPatch: { [fieldName]: cellValue }, }); } continue; } // A cell whose underlying durable work was fully satisfied from // content-addressed receipts recorded a `cached`/`reused` cell-meta // marker while the body ran (see resolveRequestsFromReceipt). Emitting // an unconditional `completed` here would clobber that marker to a // non-reused `completed`, so the run's per-column `cached` counter // reads 0 on a full-reuse rerun even though provider spend was 0. // Preserve the reuse signal so `columnStats.cached` reflects real // receipt reuse consistent with the near-zero provider spend. const cellWasReused = currentCellStatus === 'cached' && currentCellRecord?.reused === true; this.emitScopedFieldMetaUpdate({ rowId: idx, key: rowKey, tableNamespace: normalizedTableNamespace, fieldName, status: cellWasReused ? 'cached' : 'completed', stage: cellWasReused ? 'cached' : 'completed', ...(cellWasReused ? { reused: true } : {}), completedAt: Date.now(), dataPatch: shouldPersistMapCellField(fieldName) ? { [fieldName]: cellValue } : {}, }); } finishResolver(); const merged = cloneCsvAliasedRow(baseRow, computedFields); activeFieldName = null; this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, { rowId: idx, status: 'completed', stage: 'completed', provider: null, error: null, dataPatch: {}, }); const completedRow: PersistableMapRow = completedMapRowOutcome({ key: rowKey, inputIndex: rowIndex, data: this.toPersistedOutputRow(merged), ...(this.activeMapCellMeta?.get(rowKey) ? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) } : {}), }); completedRowCount += 1; if (previewRows.length < 5) { previewRows.push(this.toMaterializedOutputRow(completedRow.data)); } retainPersistedRow(completedRow, completedRowsToPersist); await enqueueIncrementalPersist(completedRow, () => { this.clearActiveMapCheckpointUpdate(rowKey, normalizedTableNamespace); updateMapFrameProgress({ completedRowKey: rowKey }); }); return COMPLETED_ROW; } catch (error) { if (isPlayRowExecutionSuspendedError(error)) { this.pendingRowEventBoundaries.push(error.boundary); updateMapFrameProgress({ pendingRowKey: rowKey, activeBoundaryId: error.boundary.boundaryId, emitEventType: 'map.suspended', }); const fieldName = rowContext.getStore()?.fieldName ?? activeFieldName; this.emitScopedFieldMetaUpdate({ rowId: idx, key: rowKey, tableNamespace: normalizedTableNamespace, fieldName, status: 'running', rowStatus: 'running', stage: 'waiting_for_event', provider: null, error: null, dataPatch: {}, }); return WAITING_ROW; } const fieldName = rowContext.getStore()?.fieldName; updateMapFrameProgress({ status: 'failed', emitEventType: 'map.failed', }); this.emitScopedFieldMetaUpdate({ rowId: idx, key: rowKey, tableNamespace: normalizedTableNamespace, fieldName: fieldName ?? activeFieldName, status: 'failed', rowStatus: 'failed', stage: 'failed', provider: null, error: this.formatRuntimeError(error), dataPatch: {}, }); throw error; } finally { finishResolver(); globalRowSlot.release(); } }; const workerCount = Math.min(rowConcurrencyLimit, items.length); const workers = Array.from({ length: workerCount }, async () => { while (firstRowError == null) { const idx = nextRowIndex; nextRowIndex += 1; if (idx >= items.length) return; try { settledResults[idx] = await runRow(idx); } catch (error) { firstRowError = error; throw error; } } }); await this.drainQueuedWork(workers); this.lastDatasetStep = datasetStep; this.activeDatasetStep = null; const settled = await Promise.allSettled(workers); const rejected = settled.find( (result): result is PromiseRejectedResult => result.status === 'rejected', ); if (rejected) { throw new FailFastMapRowsError({ cause: rejected.reason, completedRows: completedRowsToPersist, failedRows: failedRowsToPersist, }); } const missingResultIndex = settledResults.findIndex( (result) => result === undefined, ); if (missingResultIndex !== -1) { throw new Error( `Runtime map execution worker pool ended before row ${missingResultIndex} settled for ctx.dataset("${normalizedTableNamespace}").`, ); } const rowResults = settledResults as MapRowExecutionResult[]; const accountedRowIndexes = new Set(); for (const row of completedRowsToPersist) { if (typeof row.inputIndex === 'number') { accountedRowIndexes.add(row.inputIndex); } } for (const row of failedRowsToPersist) { if (typeof row.inputIndex === 'number') { accountedRowIndexes.add(row.inputIndex); } } if (this.pendingRowEventBoundaries.length > 0) { const completedRowKeys = new Set(); for (let index = 0; index < rowResults.length; index += 1) { const result = rowResults[index]; const rawItem = this.toOutputRow( items[index] as Record, ); const key = executionRowKey(rawItem, index); if (result === WAITING_ROW) { if (key) { completedRowKeys.add(key); } accountedRowIndexes.add(executionRowIndex(index)); continue; } if (key) completedRowKeys.add(key); } this.setMapFrame({ ...(this.checkpoint.mapFrames?.[mapScope.mapInvocationId] ?? { mapInvocationId: mapScope.mapInvocationId, logicalNamespace: mapScope.logicalNamespace, artifactTableNamespace: mapScope.artifactTableNamespace, status: 'suspended' as const, totalRows, completedRowKeys: [...completedRowKeys], pendingRowKeys: items.map((item, index) => executionRowKey( this.toOutputRow(item as Record), index, ), ), startedAt: Date.now(), updatedAt: Date.now(), }), status: 'suspended', completedRowKeys: [...completedRowKeys], updatedAt: Date.now(), }); const uniqueBoundaries = [ ...new Map( this.pendingRowEventBoundaries.map((boundary) => [ boundary.boundaryId, boundary, ]), ).values(), ]; const checkpointUpdates = [ ...(this.activeMapCheckpointUpdates?.values() ?? []), ].filter( (update) => (update.tableNamespace ?? normalizedTableNamespace) === normalizedTableNamespace, ); // A suspension is not observable until both terminal rows and partial // completed-cell patches cross the same writer's durability barrier. await incrementalPersistence?.checkpoint(checkpointUpdates); await incrementalPersistence?.flush(); this.pendingRowEventBoundaries = []; this.#options.onBatchComplete?.(this.checkpoint); throw new PlayExecutionSuspendedError({ kind: 'integration_event_batch', boundaries: uniqueBoundaries, }); } for (let index = 0; index < rowResults.length; index += 1) { if (rowResults[index] !== WAITING_ROW) { accountedRowIndexes.add(executionRowIndex(index)); } } if (accountedRowIndexes.size !== items.length) { const expectedRowIndexes = items.map((_item, index) => executionRowIndex(index), ); const missingRowIndexes = expectedRowIndexes.filter( (index) => !accountedRowIndexes.has(index), ); throw new Error( `Runtime map execution accounting mismatch for ctx.dataset("${normalizedTableNamespace}"): ` + `expected ${items.length} row(s), accounted for ${accountedRowIndexes.size}; ` + `missing input index(es): ${missingRowIndexes.slice(0, 10).join(', ') || 'unknown'}.`, ); } const succeededRows = rowResults.filter( (result) => result === COMPLETED_ROW, ).length; // A row is acknowledged only after its terminal sheet batch commits. The // incremental promises publish their frame updates from the commit // continuation, so this barrier also makes the terminal map event derive // from durable row truth instead of resolver completion. await incrementalPersistence?.flush(); if (emitTerminalEvent) { updateMapFrameProgress({ status: 'completed', activeBoundaryId: null, emitEventType: 'map.completed', }); } for (const failedRow of failedRowsToPersist.slice(0, 3)) { this.log( `row ${failedMapRowLogLabel(failedRow)} failed: ${failedRow.error ?? 'unknown error'}`, ); } if (failedRowsToPersist.length > 3) { this.log( `${failedRowsToPersist.length - 3} additional row failure(s) omitted from map log`, ); } this.log( `Map completed: ${succeededRows + completedRows} results (${succeededRows} succeeded, ${failedRowCount} failed, ${completedRows} duplicate keys skipped)`, ); return { completedRows: [...completedRowsToPersist].sort( comparePersistableMapRowsByInputIndex, ), failedRows: failedRowsToPersist, completedRowCount, failedRowCount, previewRows, retainedRowsComplete, }; } private stepProgramToMapDefinition( program: RuntimeStepProgram, ): MapFieldDefinition> { const definition: MapFieldDefinition> = {}; const continueOnProviderUnavailable = continuesOnProviderUnavailable(program); for (const step of program.steps) { const resolver: MapFieldResolver< Record, Record > = async ( _row: Record, _ctx: unknown, currentRow: Record, index: number, previousCell?: PreviousCell, ) => { try { return await this.executeStepProgramStep( step, currentRow, index, [step.name], previousCell, ); } catch (error) { if ( (!continueOnProviderUnavailable && step.continueOnProviderUnavailable !== true) || (!(error instanceof ProviderExhaustedError) && !isProviderUnavailable(error)) ) { throw error; } return null; } }; definition[step.name] = resolver; } Object.defineProperty(definition, STEP_PROGRAM_MAP_DEFINITION, { value: true, enumerable: false, }); return definition; } private async executeStepProgram( program: RuntimeStepProgram, row: Record, index: number, path: string[], options?: { checkpointSteps?: boolean }, ): Promise { let currentRow = cloneCsvAliasedRow(row); const produced: Record = {}; // Steps this program has not run yet. Hoisted per program, not rebuilt per // step, and shrunk as each step produces. const pendingStepNames = new Set(program.steps.map((step) => step.name)); const waterfallAttempts: Record> = {}; const continueOnProviderUnavailable = continuesOnProviderUnavailable(program); for (const step of program.steps) { const stepPath = [...path, step.name]; const stageExecution = { skipped: false }; // ADR 0019: a step's read-set is the row as it stands *now*, which // includes every earlier step's output. Sibling steps are not part of the // row's column order, so a step cell keeps an explicit array where a // dataset column resolves through the row-grain cut point. Recorded // before the step runs so the step's own cell patches re-state it. const stepRowStore = rowContext.getStore(); if (this.docflowCaptureEnabled && stepRowStore?.rowKey) { this.recordCellProvenance(stepRowStore.rowKey, stepPath.join('.'), { reads: this.cellReadRefsForRow( stepRowStore.rowKey, [currentRow], stepPath.join('.'), pendingStepNames, ), }); } const runStep = async () => options?.checkpointSteps ? await this.step( stepPath.join('.'), async () => await this.executeStepProgramStep( step, currentRow, index, stepPath, undefined, stageExecution, ), { semanticKey: stableDigest( stableStringify({ index, input: currentRow, stepPath, }), ), }, ) : await this.executeStepProgramStep( step, currentRow, index, stepPath, undefined, stageExecution, ); let value: unknown; try { value = await runStep(); } catch (error) { // A provider admission, transient upstream failure, or exhausted // managed provider account is a visible unavailable leg, not a row // abort. Persist the failure on this stage, then make its value a // normal miss so existing `runIf` waterfalls advance without every // author hand-writing error handling. if ( !continueOnProviderUnavailable || (!(error instanceof ProviderExhaustedError) && !isProviderUnavailable(error)) ) { throw error; } const unavailableAttempt = error instanceof ProviderExhaustedError ? { provider: error.provider, operation: null, code: error.code, category: 'rate_limit', retryable: true, statusCode: null, requestId: null, retryAfterMs: null, } : { provider: error.provider ?? null, operation: error.operation ?? null, code: error.code ?? null, category: error.category ?? null, retryable: error.retryable === true, statusCode: error.statusCode ?? null, requestId: error.requestId ?? null, retryAfterMs: error.retryAfterMs ?? null, }; value = null; waterfallAttempts[step.name] = { status: 'unavailable', ...unavailableAttempt, error: this.formatRuntimeError(error), }; const rowStore = rowContext.getStore(); if (rowStore) { this.emitScopedFieldMetaUpdate({ rowId: rowStore.rowId, key: rowStore.rowKey ?? null, tableNamespace: rowStore.tableNamespace ?? null, fieldName: stepPath.join('.'), status: 'failed', rowStatus: 'running', stage: 'failed', provider: error.provider, error: this.formatRuntimeError(error), dataPatch: {}, }); } produced[step.name] = value; pendingStepNames.delete(step.name); currentRow = cloneCsvAliasedRow(currentRow, { [step.name]: value }); continue; } waterfallAttempts[step.name] = stageExecution.skipped ? { status: 'skipped' } : value === null ? { status: 'no_result' } : { status: 'completed', result: value }; produced[step.name] = value; pendingStepNames.delete(step.name); const rowStore = rowContext.getStore(); const fieldName = stepPath.join('.'); const patchFieldName = runtimeSheetPatchFieldName(fieldName); if (rowStore && shouldPersistMapCellField(patchFieldName)) { const cellValue = await this.serializeCellValue(value); this.emitScopedFieldMetaUpdate({ rowId: rowStore.rowId, key: rowStore.rowKey ?? null, tableNamespace: rowStore.tableNamespace ?? null, fieldName, status: 'completed', rowStatus: 'running', stage: 'completed', provider: null, error: null, completedAt: Date.now(), dataPatch: { [patchFieldName]: cellValue }, }); } currentRow = cloneCsvAliasedRow(currentRow, { [step.name]: value }); } if (typeof program.returnResolver === 'function') { const result = await program.returnResolver(currentRow, this, index); return continueOnProviderUnavailable ? this.appendWaterfallAttempts(result, waterfallAttempts) : result; } return continueOnProviderUnavailable ? this.appendWaterfallAttempts(produced, waterfallAttempts) : produced; } private appendWaterfallAttempts( result: unknown, waterfallAttempts: Record>, ): unknown { if ( result === null || typeof result !== 'object' || Array.isArray(result) || Object.prototype.hasOwnProperty.call(result, 'waterfall_attempts') ) { return result; } return { ...(result as Record), waterfall_attempts: waterfallAttempts, }; } private async executeStepProgramStep( step: RuntimeStepProgramStep, currentRow: Record, index: number, path: string[], previousCell?: PreviousCell, stageExecution?: { skipped: boolean }, ): Promise { const resolver = step.resolver; const store = rowContext.getStore(); const nestedFieldName = path.join('.'); const runWithStepScope = async (run: () => Promise) => { if (!store) return await run(); return await rowContext.run( { ...store, fieldName: nestedFieldName, }, run, ); }; if (isRuntimeStepProgram(resolver)) { return await runWithStepScope( async () => await this.executeStepProgram(resolver, currentRow, index, path), ); } if (isRuntimeConditionalStepResolver(resolver)) { const shouldRun = await resolver.when(currentRow, index); // ADR 0019: the per-row branch this evaluator selected, written onto the // cell the decision governs. `run`/`else` is the runtime's own vocabulary // for a `runIf` conditional — the only per-row branch construct the // runtime evaluates. A branch inside authored code is invisible here and // degrades to the transform's read-set, which is the honest floor. if (this.docflowCaptureEnabled && store?.rowKey) { // The evaluator was handed the same row the resolver would get, so the // decision's read-set is the cell's read-set. Reusing the seeded trace // keeps the two consistent and costs no second pass over the row. this.recordCellProvenance(store.rowKey, nestedFieldName, { decide: { branch: shouldRun ? 'run' : 'else', reads: this.effectiveCellReads(store.rowKey, nestedFieldName), }, }); } if (!shouldRun) { const elseValue = Object.prototype.hasOwnProperty.call( resolver, 'elseValue', ) ? resolver.elseValue : null; if (store) { this.emitScopedFieldMetaUpdate({ rowId: store.rowId, key: store.rowKey ?? null, tableNamespace: store.tableNamespace ?? null, fieldName: nestedFieldName, status: 'skipped', rowStatus: 'running', stage: 'skipped', provider: null, error: null, dataPatch: {}, }); } if (stageExecution) stageExecution.skipped = true; return elseValue; } return await runWithStepScope( async () => await resolver.run(currentRow, this, index, previousCell), ); } if (typeof resolver !== 'function') { return resolver; } return await runWithStepScope( async () => await resolver(currentRow, this, index, previousCell), ); } private canUsePureJsMapFastPath( definition: MapFieldDefinition, ): boolean { if ( (definition as Record)[STEP_PROGRAM_MAP_DEFINITION] ) { return false; } return Object.values(definition).every((resolver) => { if (typeof resolver !== 'function') { return true; } const source = Function.prototype.toString.call(resolver); return ( !source.includes('.tools.execute(') && !source.includes('.runPlay(') ); }); } private async runPureFieldMap( items: T[], fieldEntries: [string, MapFieldDefinition[string]][], visibleFields: string[], tableNamespace: string, rowIdentity: (row: Record, index: number) => string, ): Promise>> { const results: Array> = []; const pureDatasetColumnSet = new Set( fieldEntries.map(([fieldName]) => fieldName), ); this.pureMapExecutionActive = true; try { for (let index = 0; index < items.length; index += 1) { const item = items[index]!; const baseRow = this.toOutputRow(item); const computedFields: Record = {}; let activeFieldName: string | null = null; // The pure path emits one consolidated patch per row, so the read // trace is stamped straight onto that patch (ADR 0019) instead of // riding the seed-and-re-state path the row loop uses. const pureRowKey = rowIdentity(baseRow, index); // Pure rows complete in microseconds, so interim per-field updates // carry no live value. Accumulate every field's data/meta patch and // emit ONE consolidated terminal row_update per row — the previous // shape emitted up to 2 events per field plus a row event (~7x per // row in a multi-column map), and terminal cell states bypass the // runner's sampling, flooding the event transport at 150k-row scale. const rowDataPatch: Record = {}; const rowCellMetaPatch: NonNullable = {}; // ADR 0019 row-grain read order, built once per row on the consolidated // patch this path already emits. // // Ungated this stays empty, which is the gate: the per-field branch // below already treats an empty order as "nothing was available to // read" and does no work, and the two `length > 0` guards on the // consolidated patches omit `rowMetaPatch` entirely. This path never // touches `activeMapCellMeta`, so it has to be stopped here or not at // all. const pureReadColumns = this.docflowCaptureEnabled ? this.rowInputReadColumns(baseRow, pureDatasetColumnSet) : []; const pureReadUpto: Record = {}; try { for (const [fieldName, resolver] of fieldEntries) { activeFieldName = fieldName; // A written column joins the order before its own cell runs, so // its position is its cut; anything the list cannot place states // an explicit one. if (pureReadColumns.length === 0) { // Nothing was available to read. } else if ( shouldPersistMapCellField(fieldName) && pureReadColumns.length < MAX_CELL_READ_REFS && !pureReadColumns.includes(fieldName) ) { pureReadColumns.push(fieldName); } else if ( Object.keys(pureReadUpto).length < MAX_ROW_READ_CELLS && !pureReadColumns.includes(fieldName) ) { pureReadUpto[fieldName] = pureReadColumns.length; } const value = await this.resolveMapFieldValue( resolver, item, this.rehydrateRowFields( cloneCsvAliasedRow(baseRow, computedFields), ), index, this.previousCellForField(baseRow, fieldName), ); computedFields[fieldName] = await this.serializeCellValue(value); if (shouldPersistMapCellField(fieldName)) { rowDataPatch[fieldName] = computedFields[fieldName]; } rowCellMetaPatch[fieldName] = { status: 'completed', stage: 'completed', completedAt: Date.now(), }; } results.push( this.toPublicOutputRow(cloneCsvAliasedRow(baseRow, computedFields)), ); activeFieldName = null; this.emitScopedRowUpdate(pureRowKey, tableNamespace, { rowId: index, status: 'completed', stage: 'completed', provider: null, error: null, dataPatch: rowDataPatch, ...(pureReadColumns.length > 0 ? { rowMetaPatch: { reads: { columns: pureReadColumns, ...(Object.keys(pureReadUpto).length > 0 ? { upto: pureReadUpto } : {}), }, }, } : {}), cellMetaPatch: rowCellMetaPatch, }); } catch (error) { this.emitScopedRowUpdate(pureRowKey, tableNamespace, { rowId: index, status: 'failed', stage: 'failed', provider: null, error: this.formatRuntimeError(error), // Carry the cells that completed before the failure — previously // they had already been emitted as individual field updates. dataPatch: rowDataPatch, ...(pureReadColumns.length > 0 ? { rowMetaPatch: { reads: { columns: pureReadColumns, ...(Object.keys(pureReadUpto).length > 0 ? { upto: pureReadUpto } : {}), }, }, } : {}), cellMetaPatch: { ...rowCellMetaPatch, [String(activeFieldName ?? '__unknown')]: { status: 'failed', stage: 'failed', error: this.formatRuntimeError(error), }, }, }); throw error; } if ((index + 1) % PURE_JS_HEARTBEAT_ROW_INTERVAL === 0) { this.pulseProgressHeartbeat(); } } } finally { this.pulseProgressHeartbeat(true); this.pureMapExecutionActive = false; } return results; } private initializeRowStates(items: readonly unknown[]): void { for (let idx = 0; idx < items.length; idx += 1) { this.rowStates.set(idx, { results: new Map(), }); } } private fixtureProviderPacingDisabled(): boolean { return ( this.#options.integrationMode === 'fixture' && this.#options.enforceFixtureProviderPacing !== true ); } private toolDispatchLane(request: ToolCallRequest): { key: string; readyCount: number; maxGroupSize: number; coalesceWindowMs: number; } { const strategy = this.#options.getBatchOperationStrategy?.(request.toolId) ?? null; if (strategy) { return { key: [ 'batch', request.toolId, request.executionAuthScopeDigest ?? '', String(strategy.toBucketKey(request.input)), ].join('\u0000'), readyCount: strategy.maxBatchSize, maxGroupSize: this.governor.policy.concurrency.toolCalls, coalesceWindowMs: this.#options.toolBatchCoalesceWindowMs ?? TOOL_BATCH_COALESCE_WINDOW_MS, }; } return { key: [ 'scalar', request.toolId, request.executionAuthScopeDigest ?? '', ].join('\u0000'), readyCount: this.governor.policy.pacing.workerToolBatchDefaultParallelism, maxGroupSize: this.governor.policy.concurrency.toolCalls, coalesceWindowMs: this.#options.toolScalarCoalesceWindowMs ?? TOOL_SCALAR_COALESCE_WINDOW_MS, }; } private wakeToolDispatcher(): void { const waiters = [...this.toolDispatcherWakeWaiters]; this.toolDispatcherWakeWaiters.clear(); for (const wake of waiters) wake(); } private toolBatchDispatcherNowMs(): number { return this.#options.toolBatchDispatcherClock?.nowMs() ?? Date.now(); } private enqueueToolCall(request: ToolCallRequest): void { if (this.toolDispatcherFailure != null) { const resolver = this.toolCallResolvers.get(request.callId); if (resolver) { resolver.reject(this.toolDispatcherFailure); this.toolCallResolvers.delete(request.callId); } return; } this.toolCallQueue.push(request); const lane = this.toolDispatchLane(request); if (!this.toolDispatchQueuedAtByLane.has(lane.key)) { this.toolDispatchQueuedAtByLane.set( lane.key, this.toolBatchDispatcherNowMs(), ); } this.wakeToolDispatcher(); } private rejectQueuedToolCalls(error: unknown): void { const queued = this.toolCallQueue; this.toolCallQueue = []; this.toolDispatchQueuedAtByLane.clear(); for (const request of queued) { const resolver = this.toolCallResolvers.get(request.callId); if (!resolver) continue; resolver.reject(error); this.toolCallResolvers.delete(request.callId); } } private async rejectUnresolvedToolCalls( requests: readonly ToolCallRequest[], error: unknown, ): Promise { await Promise.all( requests.map(async (request) => { if (!this.toolCallResolvers.has(request.callId)) return; await this.rejectToolCall(request.toolId, request, error, { persistReceiptFailure: false, }); }), ); } private takeDispatchableToolCalls( nowMs: number, blockedLaneKeys: ReadonlySet, ): { requests: ToolCallRequest[]; laneKeys: Set; nextDeadlineMs: number | null; } { if (this.toolCallQueue.length === 0) { return { requests: [], laneKeys: new Set(), nextDeadlineMs: null }; } const laneCounts = new Map(); const lanes = new Map< string, { readyCount: number; maxGroupSize: number; coalesceWindowMs: number; } >(); for (const request of this.toolCallQueue) { const lane = this.toolDispatchLane(request); laneCounts.set(lane.key, (laneCounts.get(lane.key) ?? 0) + 1); lanes.set(lane.key, lane); } const readyLaneKeys = new Set(); let nextDeadlineMs: number | null = null; for (const [laneKey, count] of laneCounts) { if (blockedLaneKeys.has(laneKey)) continue; const queuedAt = this.toolDispatchQueuedAtByLane.get(laneKey) ?? nowMs; if (!this.toolDispatchQueuedAtByLane.has(laneKey)) { this.toolDispatchQueuedAtByLane.set(laneKey, queuedAt); } const lane = lanes.get(laneKey)!; const deadline = queuedAt + lane.coalesceWindowMs; if (count >= lane.readyCount || deadline <= nowMs) { readyLaneKeys.add(laneKey); } else { nextDeadlineMs = nextDeadlineMs == null ? deadline : Math.min(nextDeadlineMs, deadline); } } const requests: ToolCallRequest[] = []; const remaining: ToolCallRequest[] = []; const selectedByLane = new Map(); for (const request of this.toolCallQueue) { const lane = this.toolDispatchLane(request); const selectedCount = selectedByLane.get(lane.key) ?? 0; if ( readyLaneKeys.has(lane.key) && requests.length < this.governor.policy.concurrency.toolCalls && selectedCount < lane.maxGroupSize ) { requests.push(request); selectedByLane.set(lane.key, selectedCount + 1); } else { remaining.push(request); } } this.toolCallQueue = remaining; const selectedLaneKeys = new Set(selectedByLane.keys()); for (const laneKey of selectedLaneKeys) { this.toolDispatchQueuedAtByLane.delete(laneKey); } return { requests, laneKeys: selectedLaneKeys, nextDeadlineMs }; } private waitForToolDispatcherWake(deadlineMs: number | null): Promise { return new Promise((resolve) => { let cancelDeadline: (() => void) | null = null; const wake = () => { cancelDeadline?.(); cancelDeadline = null; this.toolDispatcherWakeWaiters.delete(wake); resolve(); }; this.toolDispatcherWakeWaiters.add(wake); if (deadlineMs != null) { const delayMs = Math.max( 0, deadlineMs - this.toolBatchDispatcherNowMs(), ); const clock = this.#options.toolBatchDispatcherClock; if (clock) { cancelDeadline = clock.schedule(delayMs, wake); } else { const timer = setTimeout(wake, delayMs); cancelDeadline = () => clearTimeout(timer); } } }); } private async drainQueuedWork(promises: Promise[]): Promise { // One dispatcher owns ready work, fixed lane coalescing deadlines, and the // bounded in-flight registry. A row resumes after its own receipt persists; // it never waits for unrelated sibling calls from the previous column. const inFlightToolExecutions = new Set>(); const activeGroupsByLane = new Map(); const maxInFlightGroups = Math.max( 1, Math.floor( this.#options.toolDispatcherMaxInFlightGroups ?? this.governor.policy.concurrency.toolDispatchGroups, ), ); const maxInFlightGroupsPerLane = Math.max( 1, Math.floor( this.#options.toolDispatcherMaxInFlightGroupsPerLane ?? this.governor.policy.concurrency.toolDispatchGroupsPerLane, ), ); let rowsSettled = false; void Promise.allSettled(promises).then(() => { rowsSettled = true; this.wakeToolDispatcher(); }); // A pending Promise does not keep a short-lived Node runner alive. Keep one // handle open without polling scheduler state; actual work wakes the // dispatcher through queue/in-flight/row-settlement signals. const keepAlive = setInterval(() => undefined, 1_000); try { let pass = 0; while (true) { if (this.toolDispatcherFailure != null) { this.rejectQueuedToolCalls(this.toolDispatcherFailure); await Promise.allSettled([...inFlightToolExecutions]); throw this.toolDispatcherFailure; } const blockedLaneKeys = new Set( [...activeGroupsByLane] .filter(([, count]) => count >= maxInFlightGroupsPerLane) .map(([laneKey]) => laneKey), ); const dispatchable = inFlightToolExecutions.size >= maxInFlightGroups ? { requests: [] as ToolCallRequest[], laneKeys: new Set(), nextDeadlineMs: null, } : this.takeDispatchableToolCalls( this.toolBatchDispatcherNowMs(), blockedLaneKeys, ); if (dispatchable.requests.length > 0) { pass += 1; this.log(` Batch pass ${pass}`); this.log( ` Dispatcher launch: ready=${dispatchable.requests.length} ` + `queued=${this.toolCallQueue.length} ` + `in_flight_groups=${inFlightToolExecutions.size}`, ); for (const laneKey of dispatchable.laneKeys) { activeGroupsByLane.set( laneKey, (activeGroupsByLane.get(laneKey) ?? 0) + 1, ); } const tracked = this.executeBatchedToolCalls(dispatchable.requests) .catch(async (error) => { this.toolDispatcherFailure ??= error; this.rejectQueuedToolCalls(error); await this.rejectUnresolvedToolCalls( dispatchable.requests, error, ); }) .finally(() => { for (const laneKey of dispatchable.laneKeys) { const remainingGroups = (activeGroupsByLane.get(laneKey) ?? 1) - 1; if (remainingGroups > 0) { activeGroupsByLane.set(laneKey, remainingGroups); } else { activeGroupsByLane.delete(laneKey); } } inFlightToolExecutions.delete(tracked); this.wakeToolDispatcher(); }); inFlightToolExecutions.add(tracked); continue; } if ( rowsSettled && this.toolCallQueue.length === 0 && inFlightToolExecutions.size === 0 ) { break; } await this.waitForToolDispatcherWake(dispatchable.nextDeadlineMs); } } finally { clearInterval(keepAlive); } } private async resolveMapFieldValue( resolver: MapFieldResolver, row: T, fields: Record, index: number, previousCell?: PreviousCell, ): Promise { if (typeof resolver !== 'function') { return resolver; } return await resolver(row, this, fields, index, previousCell); } private toOutputRow(item: T): Record { if (item != null && typeof item === 'object' && !Array.isArray(item)) { return stripCsvProjectionMetadata(item as Record); } return { value: item }; } private toPublicOutputRow( row: Record, ): Record { const stripped = stripCsvProjectedFields(row); return Object.fromEntries( Object.entries(stripped) .filter( ([fieldName]) => shouldPersistMapCellField(fieldName) && !fieldName.startsWith('__deepline'), ) .map(([fieldName, value]) => [ fieldName, this.rehydrateCellValue(value), ]), ); } private toMaterializedOutputRow( row: Record, ): Record { const stripped = stripCsvProjectedFields(row); return Object.fromEntries( Object.entries(stripped) .filter( ([fieldName]) => shouldPersistMapCellField(fieldName) && !fieldName.startsWith('__deepline'), ) .map(([fieldName, value]) => [ fieldName, this.rehydrateCellValue(value), ]), ); } private toPersistedOutputRow( row: Record, ): Record { const stripped = toSerializableCsvAliasedRow(row); return Object.fromEntries( Object.entries(stripped).filter( ([fieldName]) => shouldPersistMapCellField(fieldName) && !fieldName.startsWith('__deepline'), ), ); } private async executeTool( key: string, toolId: string, input: Record, options?: ToolCallOptions, ): Promise { if (options?.description !== undefined) { validatePlayAuthoringField( 'ctx.tools.execute.description', options.description, ); } if (options?.force !== undefined) { validatePlayAuthoringField('ctx.tools.execute.force', options.force); } if ( options?.timeoutMs !== undefined && this.currentAuthoringContractEdition >= 2 ) { validatePlayAuthoringField( 'ctx.tools.execute.timeoutMs', options.timeoutMs, ); } if ( options?.receiptWaitMs !== undefined && this.currentAuthoringContractEdition >= 2 ) { validatePlayAuthoringField( 'ctx.tools.execute.receiptWaitMs', options.receiptWaitMs, ); } const executionScope = this.currentExecutionScope; const normalizedKey = this.normalizeContextKey(key, 'tool'); const toolCachePolicy = this.effectiveToolCallCachePolicy(options); const toolRequestIdentity = deriveToolRequestIdentity({ toolId, requestInput: input, }); const store = rowContext.getStore(); let logicalCallId: string | null = null; if (!store) { const callIndexKey = `${this.currentExecutionScope.receipt.namespace}:workflow:${normalizedKey}:${toolRequestIdentity}`; const callIndex = this.toolCallIndexByKey.get(callIndexKey) ?? 0; this.toolCallIndexByKey.set(callIndexKey, callIndex + 1); logicalCallId = stableDigest(`${callIndexKey}:${callIndex}`); } let executionAuthScopeDigest = (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null; const eventWaitHandler = (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null; const cacheableToolResult = !eventWaitHandler && !isQueryResultDatasetReadRequest(toolId, input) && !isAlwaysFreshIntegrationTool(toolId); let durableCacheKey = await this.durableToolCallCacheKey({ toolId, requestInput: input, executionAuthScopeDigest, staleAfterSeconds: toolCachePolicy.staleAfterSeconds, }); let providerIdempotencyKeyBase = await this.durableToolCallCacheKey( { toolId, requestInput: input, executionAuthScopeDigest, staleAfterSeconds: toolCachePolicy.staleAfterSeconds, }, null, ); const checkpointCacheKeys = [durableCacheKey]; const executeTool = async (context?: { leaseId?: string | null; retainExternalCallSlot?: (release: () => void) => void; }): Promise => { if (eventWaitHandler) { return this.executeIntegrationEventWaitTool( toolId, input, eventWaitHandler, ); } if (toolId === 'run_javascript') { throw new Error(DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE); } // Tool-call budget is charged by the Governor's acquireToolSlot in // callToolExecutionAPI, so each real outbound call is counted exactly once // (cached/checkpoint recoveries never acquire a slot and so never charge). if (this.pureMapExecutionActive && !store) { throw new Error( 'ctx.tools.execute() cannot run inside the pure-JS fast path. Call it directly in the map definition so the batching runtime can stay enabled.', ); } if (!store) { const directCacheKey = durableCacheKey; const cached = !cacheableToolResult ? null : toolCachePolicy.force ? null : this.getCachedToolResultCandidate( toolId, checkpointCacheKeys, 'direct', ); if (cached) { this.log(`Calling tool: ${toolId} recovered from checkpoint`); // Started and settled are paired in a finally: an uncovered throw // between them strands the tool node `running` forever in the // snapshot, which is exactly the lossy state ADR 0018 removed. this.emitToolCallLifecycle('started', toolId, normalizedKey); let settled = false; try { const recovered = await this.wrapToolExecutionResult({ toolId, status: cached.result.result == null ? 'no_result' : 'completed', result: cached.result.result, requestInput: input, execution: toolExecutionMetadataForOutcome({ kind: 'checkpoint', cacheKey: cached.cacheKey, }), }); settled = true; this.emitToolCallLifecycle('settled', toolId, normalizedKey, { outcome: 'cached', durationMs: 0, }); return recovered; } finally { if (!settled) { this.emitToolCallLifecycle('settled', toolId, normalizedKey, { outcome: 'failed', durationMs: 0, }); } } } this.log( toolCachePolicy.force ? `Calling tool: ${toolId} (force)` : `Calling tool: ${toolId}`, ); // ADR 0018: the tool node's lifecycle is an explicit typed event, not a // regex over this log line. const directCallStartedAt = Date.now(); this.emitToolCallLifecycle('started', toolId, normalizedKey); // Settled exactly once, including on paths that throw before the // explicit settle below (result wrapping, receipt-ownership failures, // authoring validation). `executeDirectToolCall` settles its own // transport failure and marks this handled. let directSettled = false; const settleDirectCall = ( outcome: 'completed' | 'no_result' | 'failed' | 'cached', error?: unknown, ) => { if (directSettled) return; directSettled = true; this.emitToolCallLifecycle('settled', toolId, normalizedKey, { outcome, durationMs: Date.now() - directCallStartedAt, ...(error !== undefined ? { error: this.formatRuntimeError(error) } : {}), }); }; try { const directReceiptLeaseId = context?.leaseId?.trim() || null; if (!logicalCallId) { throw new Error( 'Direct tool execution requires a stable logical call identity.', ); } const physicalDirectKey = cacheableToolResult ? directCacheKey : `${buildDurableToolReceiptPrefix({ orgId: this.#options.orgId ?? 'unknown-org', toolId, })}${stableDigest(`${this.currentRunId}:${logicalCallId}:always-fresh`)}`; const execution = await this.executeDirectToolCall({ toolId, callKey: normalizedKey, startedAt: directCallStartedAt, input, options: { durableCallReceiptKey: physicalDirectKey, playNodeScope: buildPlayNodeScope({ toolId, callKey: normalizedKey, }), executionAuthScopeDigest, providerIdempotencyReceiptKey: cacheableToolResult ? providerIdempotencyKeyBase : physicalDirectKey, providerIdempotencyKey: cacheableToolResult ? this.providerIdempotencyKeyForToolCall({ cacheKey: providerIdempotencyKeyBase, force: toolCachePolicy.force, leaseId: directReceiptLeaseId, logicalCallId, }) : physicalDirectKey, timeoutMs: resolveToolRuntimeTimeoutMs( toolId, options?.timeoutMs, this.currentAuthoringContractEdition, ), retainToolSlot: context?.retainExternalCallSlot, ...(directReceiptLeaseId && (this.#options.heartbeatRuntimeStepReceipts || this.#options.getRuntimeStepReceipt || this.#options.getRuntimeStepReceipts) ? { beforeProviderCall: async () => { await this.assertRuntimeToolReceiptOwnership([ { callId: 'direct', cacheKey: directCacheKey, receiptKey: directCacheKey, receiptLeaseId: directReceiptLeaseId, rowId: -1, toolId, input, }, ]); }, } : {}), }, }); const wrapped = await this.wrapToolExecutionResult({ toolId, status: execution.status, jobId: execution.jobId, result: execution.result, toolResponse: execution.toolResponse, metadata: execution.metadata, meta: execution.meta, requestInput: input, execution: toolExecutionMetadataForOutcome({ kind: 'live', cacheKey: directCacheKey, }), }); settleDirectCall(toolAttemptOutcomeForResult(wrapped)); if (cacheableToolResult) { this.cacheToolResult(toolId, directCacheKey, wrapped, 'direct'); this.#options.onBatchComplete?.(this.checkpoint); } return wrapped; } catch (error) { settleDirectCall('failed', error); throw error; } finally { settleDirectCall('failed'); } } const rowId = store.rowId; const fieldName = store.fieldName; const callId = [ executionScope.receipt.namespace, store.tableNamespace?.trim() || 'map', store.rowKey?.trim() || String(rowId), normalizedKey, toolId, toolRequestIdentity, ].join(':'); if (this.toolCallResolvers.has(callId)) { throw new Error( `ctx.tools.execute("${normalizedKey}") was called more than once concurrently for the same row. ` + 'Use a unique id for each row-scoped tool call.', ); } const toolResultCacheKey = durableCacheKey; const cached = !cacheableToolResult ? null : toolCachePolicy.force ? null : this.getCachedToolResultCandidate(toolId, checkpointCacheKeys); if (cached) { this.log(` Row ${rowId} ${toolId}: recovered from checkpoint`); return await this.wrapToolExecutionResult({ toolId, status: cached.result.result == null ? 'no_result' : 'completed', result: cached.result.result, requestInput: input, execution: toolExecutionMetadataForOutcome({ kind: 'checkpoint', cacheKey: cached.cacheKey, }), }); } // ADR 0018: record this leg before it runs. `producer` still names the // last leg (the usual cascade winner); `producers` keeps every leg, which // is the only durable record a losing leg ever gets — a failed tool call // writes no receipt at all. const producerAttemptStartedAt = Date.now(); // Deliberately minimal: `displayName` is derivable from `toolId`, and // this record is retained per row until the map's persistence barrier. const producerAttempt: PlayCellProducerAttempt = { kind: 'tool', id: normalizedKey, toolId, at: producerAttemptStartedAt, outcome: 'running', }; const settleProducerAttempt = ( outcome: PlayCellProducerAttempt['outcome'], ) => { producerAttempt.outcome = outcome; producerAttempt.durationMs = Date.now() - producerAttemptStartedAt; }; const pendingToolCall = new Promise((resolve, reject) => { this.toolCallResolvers.set(callId, { resolve, reject }); this.emitScopedFieldMetaUpdate({ rowId, key: store.rowKey ?? null, tableNamespace: store.tableNamespace ?? null, fieldName, status: 'running', rowStatus: 'running', stage: toolId, provider: null, error: null, producer: { kind: 'tool', id: normalizedKey, toolId, displayName: displayNameFromProducerId(toolId), }, producerAttempt, dataPatch: {}, }); const timeoutMs = resolveToolRuntimeTimeoutMs( toolId, options?.timeoutMs, this.currentAuthoringContractEdition, ); this.enqueueToolCall({ callId, cacheKey: toolResultCacheKey, providerIdempotencyKeyBase, cacheable: cacheableToolResult, receiptKey: cacheableToolResult ? durableCacheKey : null, executionAuthScopeDigest, force: toolCachePolicy.force, forceFailedRefresh: toolCachePolicy.forceFailedRefresh, rowId, fieldName, contextKey: normalizedKey, toolId, input, ...(timeoutMs !== undefined ? { timeoutMs } : {}), ...(options?.receiptWaitMs !== undefined ? { receiptWaitMs: options.receiptWaitMs } : {}), tableNamespace: store.tableNamespace, rowKey: store.rowKey ?? null, description: normalizeStepDescription(options?.description), }); }); return await pendingToolCall.then( (value) => { settleProducerAttempt(toolAttemptOutcomeForResult(value)); return value; }, (error: unknown) => { settleProducerAttempt('failed'); throw error; }, ); }; if (store || !cacheableToolResult) { return await executeTool(); } for ( let authScopeAttempt = 0; authScopeAttempt < 2; authScopeAttempt += 1 ) { try { const toolRetryPolicy = await this.#options .getToolRetryPolicy?.(toolId, input) .catch(() => null); return await this.executeWithRuntimeReceipt( 'tool', normalizedKey, this.currentRunId, { receiptKey: durableCacheKey, semanticKey: toolRequestIdentity, force: toolCachePolicy.force, // Missing metadata is intentionally lock-free. Only explicitly // dangerous non-idempotent side effects enter the fence path. requiresExecutionLock: toolRetryPolicy?.requiresExecutionFence === true, executionLockTtlMs: Math.min( 600_000, resolveToolRuntimeTimeoutMs( toolId, options?.timeoutMs, this.currentAuthoringContractEdition, ) ?? 300_000, ), staleAfterSeconds: toolCachePolicy.staleAfterSeconds, onClaimedResult: (output, receiptKey) => markToolExecuteResultExecutionOutcome(output, { kind: 'live', receiptKey, }), onRecovered: (output, _receipt, source) => markToolExecuteResultExecutionOutcome( output, toolExecutionOutcomeForDurableReceipt({ source, receiptKey: durableCacheKey, }), ), shouldPersistFailure: (error) => !(error instanceof ToolExecuteAuthScopeChangedError), markRunningBeforeExecute: false, execute: ({ leaseId, retainExternalCallSlot }) => executeTool({ leaseId, retainExternalCallSlot }), runningReceiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts( options?.receiptWaitMs !== undefined ? { max_wait_ms: options.receiptWaitMs } : input, ), }, ); } catch (error) { if ( !(error instanceof ToolExecuteAuthScopeChangedError) || authScopeAttempt > 0 ) { throw error; } // Completed-receipt cache misses never publish a running receipt. The // old auth-scope attempt therefore has no ownership metadata to clean // up; re-key under the refreshed credential scope and retry once. // Make the bounded auth-scope re-claim observable in run logs instead // of a silent continuation: the credential identity changed after the // receipt key was prepared, so we evict the cached digest, re-resolve, // and re-claim a fresh receipt under the new scope before retrying once. this.log( `ctx.tools.execute(${toolId}): auth_scope_changed_reclaim ` + `(label: ${normalizedKey}); credential identity changed mid-run, ` + `re-resolving auth scope and re-claiming a fresh receipt under the ` + `new scope (attempt ${authScopeAttempt + 1}/2).`, ); this.invalidateToolAuthScopeDigest(toolId); executionAuthScopeDigest = (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null; durableCacheKey = await this.durableToolCallCacheKey({ toolId, requestInput: input, executionAuthScopeDigest, staleAfterSeconds: toolCachePolicy.staleAfterSeconds, }); providerIdempotencyKeyBase = await this.durableToolCallCacheKey( { toolId, requestInput: input, executionAuthScopeDigest, staleAfterSeconds: toolCachePolicy.staleAfterSeconds, }, null, ); checkpointCacheKeys[0] = durableCacheKey; } } throw new ToolExecuteAuthScopeChangedError(); } private async executeIntegrationEventWaitTool( toolId: string, input: Record, handler: IntegrationEventWaitHandler, ): Promise { this.assertInlineChildContract('suspending_child'); if (!this.#options.durableBoundaries) { throw new Error(`${toolId} requires durable play boundaries.`); } const rowScope = rowContext.getStore(); const eventContext = { playId: this.#options.playId, runId: this.#options.runId, workflowId: this.#options.workflowId, orgId: this.#options.orgId, executorToken: this.#options.executorToken, }; const basePreparedBoundary = await handler.prepare({ payload: input, context: eventContext, }); const preparedBoundary = rowScope ? { ...basePreparedBoundary, boundaryId: `map-row-${stableDigest( stableStringify({ tableNamespace: rowScope.tableNamespace, rowKey: rowScope.rowKey, fieldName: rowScope.fieldName, boundaryId: basePreparedBoundary.boundaryId, eventKey: basePreparedBoundary.eventKey, }), )}`, } : basePreparedBoundary; const existing = this.checkpoint.resolvedBoundaries?.[preparedBoundary.boundaryId]; if (existing?.kind === 'integration_event' && 'output' in existing) { this.log( `Integration event ${preparedBoundary.boundaryId}: recovered response from checkpoint`, ); // Wrapped like every other tool result (toolOutput.raw, getters) — the // non-suspending runtimes (workers' synthetic wait interception) hand // plays a wrapped ToolExecuteResult, so the resume replay must too. return await this.wrapToolExecutionResult({ toolId, status: 'completed', result: existing.output, execution: toolExecutionMetadataForOutcome({ kind: 'checkpoint', cacheKey: `integration_event:${preparedBoundary.boundaryId}`, }), }); } const boundary = await handler.arm({ payload: input, context: eventContext, boundary: preparedBoundary, }); this.log( `Armed ${handler.provider} integration event wait: ${boundary.eventKey}`, ); this.checkpoint.resolvedBoundaries = { ...(this.checkpoint.resolvedBoundaries ?? {}), [boundary.boundaryId]: { kind: 'integration_event', eventKey: boundary.eventKey, timeoutMs: boundary.timeoutMs, provider: boundary.provider, toolId: boundary.toolId, ...(rowScope ? { scope: { type: 'map_row' as const, tableNamespace: rowScope.tableNamespace, rowKey: rowScope.rowKey, rowIndex: rowScope.rowId, fieldName: rowScope.fieldName, }, } : { scope: { type: 'workflow' as const } }), ...(boundary.messageRef ? { messageRef: boundary.messageRef } : {}), }, }; this.#options.onBatchComplete?.(this.checkpoint); if (rowScope) { throw new PlayRowExecutionSuspendedError({ boundaryId: boundary.boundaryId, eventKey: boundary.eventKey, timeoutMs: boundary.timeoutMs, }); } throw new PlayExecutionSuspendedError({ kind: 'integration_event', boundaryId: boundary.boundaryId, eventKey: boundary.eventKey, timeoutMs: boundary.timeoutMs, }); } async runPlay( key: string, playRef: string | { playName?: string; name?: string }, input: Record, options?: PlayCallOptions, ): Promise { if ( arguments.length === 3 && typeof playRef === 'object' && playRef !== null ) { throw new Error( 'ctx.runPlay(...) signature is ctx.runPlay(key, playRef, input, { description }). Add a stable call key as the first argument.', ); } const resolvedName = typeof playRef === 'string' ? playRef : playRef && typeof playRef.playName === 'string' ? playRef.playName : playRef && typeof playRef.name === 'string' ? playRef.name : ''; if (!resolvedName.trim()) { throw new Error('ctx.runPlay(...) requires a resolvable play name.'); } validatePlayAuthoringField('ctx.runPlay.playRef', playRef); validatePlayAuthoringField('ctx.runPlay.input', input); assertNoSecretTaint(input, 'ctx.runPlay input'); validatePlayAuthoringField('ctx.runPlay.key', key); if (options?.description !== undefined) { validatePlayAuthoringField( 'ctx.runPlay.options.description', options.description, ); } const normalizedKey = this.normalizeContextKey(key, 'runPlay'); if (!this.#options.resolvePlay) { throw new Error( 'ctx.runPlay(...) is unavailable because no play resolver was configured.', ); } const resolvedPlay = await this.#options.resolvePlay(resolvedName); if (!resolvedPlay) { throw new Error( `Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`, ); } const childCompatibilitySnapshot = resolvedPlay.contractSnapshot?.compatibility ?? resolvedPlay.artifact?.compatibility ?? buildPlayContractCompatibility(); const childCompatibility = normalizePlayContractCompatibility( childCompatibilitySnapshot, ); const childToolErrorSchemaVersion = childCompatibility.toolErrorSchemaVersion; const childToolResponseContract = childCompatibility.toolResponseContract; const childToolResponseReceiptRevision = childCompatibility.toolResponseReceiptRevision; const childExecutionDecision = resolveChildExecutionStrategy({ pipeline: resolvedPlay.staticPipeline, timeoutMs: options?.timeoutMs, hasExplicitTimeout: options != null && Object.hasOwn(options, 'timeoutMs'), execution: options?.execution, childPlayName: resolvedName, }); this.log( `ctx.runPlay(${normalizedKey}): ${childExecutionDecision.strategy} (${childExecutionDecision.reason})`, ); const compositionNamespace = this.inlineChildCompositionNamespace( resolvedName, normalizedKey, input, ); const inlineChildGovernor = await this.currentExecutionGovernor.forkInlineChild({ childPlayName: resolvedName, childRunId: compositionNamespace, }); const rowStore = rowContext.getStore(); const producer = { kind: 'play' as const, id: normalizedKey, playId: resolvedName, displayName: displayNameFromProducerId(resolvedName), runId: compositionNamespace, }; try { if (rowStore) { this.emitScopedFieldMetaUpdate({ rowId: rowStore.rowId, key: rowStore.rowKey ?? null, tableNamespace: rowStore.tableNamespace ?? null, fieldName: rowStore.fieldName, status: 'running', rowStatus: 'running', stage: resolvedName, provider: 'deepline_native', error: null, producer, dataPatch: {}, }); } // Inline composition has no child run, launch request, terminal poll, or // scheduler slot. It shares the caller's execution and tool governors. const childExecutionScope = deriveChildRunExecutionScope( this.currentExecutionScope, { placement: 'inline', runId: compositionNamespace, playId: resolvedName, receiptNamespace: compositionNamespace, }, ); this.inlineChildAggregates.total += 1; try { // A scalar child is a normal function call on the parent's context. // Async-local identity changes receipt and recursion scope without // allocating another PlayContext, tool queue, drain loop, resource // governor, or receipt client. Calls from every concurrent child // therefore reach the same proven parent batching path. const result = await inlineCompositionContext.run( { context: this, executionScope: childExecutionScope, governor: inlineChildGovernor, playName: resolvedName, staticPipeline: resolvedPlay.staticPipeline ?? null, toolErrorSchemaVersion: childToolErrorSchemaVersion, toolResponseContract: childToolResponseContract, toolResponseReceiptRevision: childToolResponseReceiptRevision, authoringContractEdition: childCompatibility.authoringContractEdition, }, () => this.executeResolvedPlay(resolvedPlay, this, input), ); this.inlineChildAggregates.ok += 1; if (rowStore) { this.emitScopedFieldMetaUpdate({ rowId: rowStore.rowId, key: rowStore.rowKey ?? null, tableNamespace: rowStore.tableNamespace ?? null, fieldName: rowStore.fieldName, status: 'completed', rowStatus: 'running', stage: resolvedName, provider: 'deepline_native', error: null, producer, dataPatch: {}, }); } this.recordPlayCallStep({ playId: resolvedName, execution: options?.execution === 'inline' ? 'inline' : undefined, description: options?.description, }); return result as TOutput; } catch (childError) { this.recordInlineChildFailure(resolvedName, childError); throw childError; } } catch (error) { if (isPlayExecutionSuspendedError(error)) { throw error; } if (rowStore) { this.emitScopedFieldMetaUpdate({ rowId: rowStore.rowId, key: rowStore.rowKey ?? null, tableNamespace: rowStore.tableNamespace ?? null, fieldName: rowStore.fieldName, status: 'failed', rowStatus: 'running', stage: resolvedName, provider: 'deepline_native', error: this.formatRuntimeError(error), producer, dataPatch: {}, }); } throw error; } } /** * Replay-stable receipt namespace for primitives executed by one inline * `ctx.runPlay` invocation. Format * `child:#@`: * * - `callKey` is the author's normalized `ctx.runPlay(key, ...)` call key, * so distinct call sites never collide. * - `invocationDigest` hashes the resolved child input and inherited row * scope. The row scope keeps concurrent map invocations isolated; the * input prevents two different people at an otherwise identical caller * row from ever recovering one another's child step or fetch receipt. * * This is not a `runPlay` result cache: `runPlay` always executes the child. * It only scopes the child's own durable step and fetch receipts. Parent run * identity is intentionally excluded, allowing semantically identical child * work to recover across retries and parent runs without permitting a * different child input to reuse a result. See ADR 0013. */ private inlineChildCompositionNamespace( childPlayName: string, normalizedKey: string, input: Record, ): string { const rowScope = rowContext.getStore(); const invocationScope = buildDurableRunPlayInvocationScope({ childPlayName, input, rowScope: rowScope ? { tableNamespace: rowScope.tableNamespace ?? null, rowKey: rowScope.rowKey ?? null, rowId: rowScope.rowKey ? null : rowScope.rowId, fieldName: rowScope.fieldName ?? null, } : null, }); const namespace = `child:${childPlayName}#${normalizedKey}@${invocationScope}`; if (rowScope) { const namespaces = (rowScope.inlineChildInvocationNamespaces ??= new Set()); namespaces.add(namespace); if (namespaces.size > MAX_INLINE_CHILD_INVOCATIONS_PER_ROW) { throw new Error( `ctx.runPlay("${childPlayName}") exceeded the inline child invocation cap ` + `(${MAX_INLINE_CHILD_INVOCATIONS_PER_ROW} per row). A resolver is calling ` + 'runPlay in an unbounded loop; give the batch ONE dataset child at play level instead.', ); } } return namespace; } private recordInlineChildFailure( childPlayName: string, error: unknown, ): void { this.inlineChildAggregates.failed += 1; if ( this.inlineChildAggregates.failures.length < MAX_INLINE_CHILD_FAILURE_DETAIL ) { this.inlineChildAggregates.failures.push({ childPlayName, error: this.formatRuntimeError(error), }); } } /** * Snapshot of inline child aggregates for the single-writer progress event. * Read-only; never mutates. Omitted entirely when no inline child has run so * plays without child composition keep byte-identical progress events. */ private inlineChildAggregateEventFields(): { childrenTotal?: number; childrenOk?: number; childrenFailed?: number; } { if (this.inlineChildAggregates.total === 0) return {}; return { childrenTotal: this.inlineChildAggregates.total, childrenOk: this.inlineChildAggregates.ok, childrenFailed: this.inlineChildAggregates.failed, }; } /** * Extract a list from a tool result. * e.g. ctx.extractList(result, 'people', ['first_name', 'last_name', 'email']) */ extractList( result: unknown, listPath: string, fields?: string[], ): Record[] { if (result == null || typeof result !== 'object') return []; let list: unknown = result; for (const key of listPath.split('.')) { if (list == null || typeof list !== 'object') return []; list = (list as Record)[key]; } if (!Array.isArray(list)) return []; if (!fields || fields.length === 0) { return list.filter( (item): item is Record => item != null && typeof item === 'object', ); } return list .filter( (item): item is Record => item != null && typeof item === 'object', ) .map((item) => { const picked: Record = {}; for (const field of fields) { if (field in item) picked[field] = item[field]; } return picked; }); } log(msg: string): void { assertNoSecretTaint(msg, 'ctx.log'); const line = `[${new Date().toISOString()}] ${this.secretRedactor.redactRegisteredSecrets(msg)}`; this.logBuffer.push(line); this.#options.onLog?.(line); if (this.#options.verbose) console.log(line); } async sleep(ms: number): Promise { this.assertInlineChildContract('suspending_child'); const delayMs = this.currentAuthoringContractEdition >= 2 ? (validatePlayAuthoringField('ctx.sleep.ms', ms), ms) : Math.max(0, Math.round(ms)); if (this.#options.durableBoundaries) { const boundaryId = this.durableBoundaryId( `sleep-${this.sleepBoundaryIndex}-${delayMs}`, ); this.sleepBoundaryIndex += 1; const existing = this.checkpoint.resolvedBoundaries?.[boundaryId]; if (existing?.kind === 'sleep' && existing.completedAt !== undefined) { return; } this.checkpoint.resolvedBoundaries = { ...(this.checkpoint.resolvedBoundaries ?? {}), [boundaryId]: { kind: 'sleep', delayMs, }, }; this.#options.onBatchComplete?.(this.checkpoint); throw new PlayExecutionSuspendedError({ kind: 'sleep', boundaryId, delayMs, }); } return new Promise((resolve) => setTimeout(resolve, delayMs)); } async fetch( key: string, input: string | URL, init: PlaySecretAwareRequestInit | SecretAwareRequestInit = {}, options?: FetchOptions, ): Promise { validatePlayAuthoringField('ctx.fetch.key', key); const normalizedKey = this.normalizeContextKey(key, 'fetch'); const rowStore = rowContext.getStore(); const rowFetchScope = rowStore ? { tableNamespace: rowStore.tableNamespace ?? null, rowKey: rowStore.rowKey ?? null, rowId: rowStore.rowKey ? null : rowStore.rowId, fieldName: rowStore.fieldName ?? null, } : null; const url = input.toString(); const parsedUrl = new URL(url); const urlContainsResolvedSecret = this.secretRedactor.containsRegisteredSecret(url, { includeEncoded: true, minimumLength: 4, }) || [...parsedUrl.searchParams.values()].some((value) => this.secretRedactor.matchesRegisteredSecret(value), ); if (valueContainsSecret(input) || urlContainsResolvedSecret) { throw new Error( 'ctx.fetch does not allow secrets in the URL. Use an approved secret auth helper or request body.', ); } if (valueContainsSecret(init.body)) { throw new Error( 'ctx.fetch does not allow opaque secret values in the body. Await ctx.secrets.get(...) before constructing the body.', ); } const requestBody = typeof init.body === 'string' ? init.body : null; const bodyContainsResolvedSecret = requestBody !== null && this.secretRedactor.containsRegisteredSecret(requestBody, { includeEncoded: true, }); if (bodyContainsResolvedSecret && parsedUrl.protocol !== 'https:') { throw new Error( 'ctx.fetch with a resolved secret in the request body requires an https:// URL. Customer secrets may only leave Deepline over TLS.', ); } const receiptBody = bodyContainsResolvedSecret ? this.secretRedactor.redactRegisteredSecrets(requestBody) : requestBody; const rawHeaders = normalizeFetchHeaders(init.headers); if ( valueContainsSecret(init.headers) || Object.values(rawHeaders).some( (value) => this.secretRedactor.matchesRegisteredSecret(value) || this.secretRedactor.containsRegisteredSecret(value, { minimumLength: 4, }), ) ) { throw new Error( 'ctx.fetch does not allow raw secret headers. Use ctx.secrets.bearer(...) or ctx.secrets.header(...).', ); } let secretAuth: SecretAuthInput | undefined; if (init.auth !== undefined) { if (!isSecretAuthInput(init.auth)) { throw new Error('ctx.fetch auth must come from ctx.secrets.'); } secretAuth = init.auth; } // Secret handles are deliberately resolved at the last possible moment, so // plaintext never lands in durable keys, receipts, map rows, or generic tool // payloads. The one place a customer secret is allowed to leave Deepline is // the requested auth header, and that transport must be TLS. assertSecretAuthUsesTls(secretAuth, input, 'ctx.fetch'); const secretHeaderMarkers = secretAuthHeaderMarkers(secretAuth); const execution = this.executeWithRuntimeReceipt( 'fetch', normalizedKey, this.currentRunId, { semanticKey: stableDigest( stableStringify({ method: (init.method ?? 'GET').toUpperCase(), url, body: receiptBody, safeHeaders: { ...normalizeFetchHeaders(init.headers), ...secretHeaderMarkers, }, row: rowFetchScope, }), ), staleAfterSeconds: options?.staleAfterSeconds, execute: async ({ retainExternalCallSlot }) => { const method = (init.method ?? 'GET').toUpperCase(); const secretHeaders = await this.resolveSecretAuth(secretAuth); const headers: Record = { ...normalizeFetchHeaders(init.headers), ...secretHeaders, }; const fetchInit = { ...init, headers }; delete fetchInit.auth; const boundaryId = this.durableBoundaryId( `fetch-${stableDigest( stableStringify({ url, method, headers: { ...normalizeFetchHeaders(init.headers), ...secretHeaderMarkers, }, body: receiptBody, row: rowFetchScope, }), )}`, ); const existing = this.checkpoint.resolvedBoundaries?.[boundaryId]; if (existing?.kind === 'fetch' && 'output' in existing) { this.log(`ctx.fetch(${url}): recovered response from checkpoint`); if (this.durableDirectToolResultsBackedByReceipts) { // The outer durable receipt is the replay authority in hosted // runtimes. A legacy checkpoint fetch may seed that receipt // once, but it must not remain as a second full response-body // cache for the lifetime of a large map. delete this.checkpoint.resolvedBoundaries?.[boundaryId]; } return existing.output as PlayFetchResponse; } if (!['GET', 'HEAD', 'OPTIONS'].includes(method)) { const hasIdempotencyKey = headers['idempotency-key'] !== undefined || headers['x-idempotency-key'] !== undefined; if (!hasIdempotencyKey) { throw new Error( `ctx.fetch(${method} ${url}) needs an Idempotency-Key header. Durable plays can replay after waits/retries; add an idempotency key or wrap the side effect in a Deepline integration tool.`, ); } } // ctx.fetch is arbitrary customer-directed egress. Pace it through // the same generic_http_request lane as the explicit Generic HTTP // integration tool so switching syntax cannot bypass per-org limits. const egressSlot = await this.resourceGovernor.acquireTool({ orgId: this.#options.orgId ?? null, providerResourceKey: CTX_FETCH_EGRESS_TOOL_ID, toolId: CTX_FETCH_EGRESS_TOOL_ID, }); retainExternalCallSlot(() => egressSlot.release()); const finishFetchObservation = rowStore?.mapStallObserver?.fetchStarted( rowStore.mapStallResolverToken, ); const fetchDeadline: CtxFetchDeadline = { startedAt: Date.now(), headersMs: positiveTimeoutMs( this.#options.ctxFetchTimeouts?.headersMs, CTX_FETCH_HEADERS_TIMEOUT_MS, ), bodyMs: positiveTimeoutMs( this.#options.ctxFetchTimeouts?.bodyMs, CTX_FETCH_BODY_TIMEOUT_MS, ), totalMs: positiveTimeoutMs( this.#options.ctxFetchTimeouts?.totalMs, CTX_FETCH_TOTAL_TIMEOUT_MS, ), }; let response: Response | null = null; let bodyText: string | null = null; try { const canRetryTransport = ['GET', 'HEAD', 'OPTIONS'].includes( method, ); const { safePublicFetch } = await loadSafeFetch(); for ( let attempt = 1; attempt <= FETCH_TRANSPORT_MAX_ATTEMPTS; attempt += 1 ) { const attemptController = new AbortController(); try { const attemptInit = { ...fetchInit, signal: init.signal ? AbortSignal.any([init.signal, attemptController.signal]) : attemptController.signal, }; response = await runCtxFetchPhase({ callerSignal: init.signal, controller: attemptController, deadline: fetchDeadline, phase: 'headers', run: () => safePublicFetch(url, attemptInit, { fetchImpl: this.#options.fetchImpl, sensitiveHeaders: Object.keys(secretHeaderMarkers), stripHeadersOnCrossOriginRedirect: true, }), }); bodyText = await readCtxFetchBody({ callerSignal: init.signal, controller: attemptController, deadline: fetchDeadline, response, }); break; } catch (error) { void response?.body?.cancel(error).catch(() => undefined); response = null; bodyText = null; if (error instanceof CtxFetchTimeoutError) throw error; if (init.signal?.aborted) { throw init.signal.reason ?? error; } if (isUnsafeOutboundUrlError(error)) { throw error; } const message = error instanceof Error ? error.message : String(error); if ( canRetryTransport && attempt < FETCH_TRANSPORT_MAX_ATTEMPTS ) { this.log( `ctx.fetch(${method} ${url}) transport failed on attempt ${attempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}; retrying: ${message}`, ); await sleepWithinCtxFetchDeadline({ callerSignal: init.signal, deadline: fetchDeadline, delayMs: FETCH_TRANSPORT_RETRY_DELAY_MS * attempt, }); continue; } throw new Error( `ctx.fetch(${method} ${url}) failed on attempt ${attempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}: ${message}`, ); } } if (!response) { throw new Error( `ctx.fetch(${method} ${url}) failed before receiving a response.`, ); } if (bodyText === null) { throw new Error( `ctx.fetch(${method} ${url}) failed while reading the response body.`, ); } const redactedBodyText = this.secretRedactor.redactString(bodyText); const output: PlayFetchResponse = { ok: response.ok, status: response.status, statusText: response.statusText, url: response.url, headers: this.secretRedactor.redact( Object.fromEntries(response.headers.entries()), ) as Record, bodyText: redactedBodyText, json: this.secretRedactor.redactKnownSecrets( parseJsonOrNull(bodyText), ), }; if (!this.durableDirectToolResultsBackedByReceipts) { this.checkpoint.resolvedBoundaries = { ...(this.checkpoint.resolvedBoundaries ?? {}), [boundaryId]: { kind: 'fetch', url, method, output, completedAt: Date.now(), }, }; this.#options.onBatchComplete?.(this.checkpoint); } return output; } finally { finishFetchObservation?.(); } }, }, ); return execution; } async step( key: string, run: () => T | Promise, options?: RuntimeStepOptions, ): Promise { validatePlayAuthoringField('ctx.step.id', key); validateOptionalPlayAuthoringField( 'ctx.step.semanticKey', options?.semanticKey, ); const normalizedKey = this.normalizeContextKey(key, 'step'); if (!normalizedKey.trim()) { throw new Error('ctx.step(key, fn) requires a non-empty stable step id.'); } const rowStore = rowContext.getStore(); const scope = rowStore ? `row-${rowStore.rowId}` : 'workflow'; const callIndexKey = `${this.currentExecutionScope.receipt.namespace}:${scope}:${normalizedKey}`; const callIndex = this.stepCallIndexByKey.get(callIndexKey) ?? 0; this.stepCallIndexByKey.set(callIndexKey, callIndex + 1); const boundarySuffix = callIndex === 0 ? '' : `:${callIndex}`; const boundaryId = this.durableBoundaryId( `step-${scope}:${normalizedKey}${boundarySuffix}`, ); const executeStep = async (): Promise => { const existing = this.checkpoint.resolvedBoundaries?.[boundaryId]; if (existing?.kind === 'step' && 'output' in existing) { this.log( `ctx.step(${normalizedKey}): recovered result from checkpoint`, ); return existing.output as T; } const output = await run(); assertJsonSerializableStepOutput(normalizedKey, output); this.checkpoint.resolvedBoundaries = { ...(this.checkpoint.resolvedBoundaries ?? {}), [boundaryId]: { kind: 'step', stepId: normalizedKey, output, completedAt: Date.now(), }, }; this.#options.onBatchComplete?.(this.checkpoint); return output; }; return this.executeWithRuntimeReceipt( 'step', normalizedKey, this.currentRunId, { semanticKey: rowStore ? stableDigest( stableStringify({ scope: 'row', tableNamespace: rowStore.tableNamespace ?? null, rowKey: rowStore.rowKey ?? null, rowId: rowStore.rowKey ? null : rowStore.rowId, fieldName: rowStore.fieldName ?? null, callIndex, // Row scope prevents unrelated map cells from sharing a // checkpoint. A supplied semantic key identifies the actual // work (for example, ctx.runSteps hashes its current input), // so it must refine row scope rather than being discarded. semanticKey: options?.semanticKey ?? null, }), ) : options?.semanticKey, staleAfterSeconds: options?.staleAfterSeconds, force: this.#options.cachePolicy?.forceStepRefresh === true, markSkipped: (output) => { assertJsonSerializableStepOutput(normalizedKey, output); }, execute: executeStep, }, ); } getLogs(): string[] { return this.logBuffer; } getCheckpoint(): PlayCheckpoint { return this.checkpoint; } getSteps(): PlayStep[] { return this.steps; } recordStep(step: PlayStep): void { const isSubstepType = step.type === 'waterfall' || step.type === 'tool' || step.type === 'run_javascript'; const targetDataset = this.activeDatasetStep ?? this.lastDatasetStep; if (targetDataset && isSubstepType) { targetDataset.substeps.push(step); } else { this.steps.push(step); } } recordReturn(outputRows: number): void { this.lastDatasetStep = null; // No more substeps expected this.steps.push({ type: 'return', outputRows }); } getStats(): Record { return { rowsProcessed: Math.max(this.rowStates.size, this.processedRowCount), }; } // ——— Batched tool call execution ——— private startPendingToolReceiptHeartbeat( requests: ToolCallRequest[], claimsEstablishedExecutionFence: boolean, ): { stop: () => void; leaseLost: () => unknown | null; confirmOwned: () => Promise; assertOwned: () => void; } | null { const heartbeatReceipts = this.#options.heartbeatRuntimeStepReceipts; const ownedRequests = requests.filter( (request) => request.receiptKey && request.receiptLeaseId, ); if (!heartbeatReceipts || ownedRequests.length === 0) return null; const earliestLeaseExpiry = ownedRequests.reduce( (earliest, request) => { const expiresAt = request.receiptLeaseExpiresAt ?? null; if (!expiresAt) return earliest; if (!earliest) return expiresAt; return Date.parse(expiresAt) < Date.parse(earliest) ? expiresAt : earliest; }, null, ); let leaseLost: unknown | null = null; let ownershipConfirmedUntilMs = Number.isFinite( Date.parse(earliestLeaseExpiry ?? ''), ) ? Date.parse(earliestLeaseExpiry!) : Date.now() + PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS; const firstOwnedRequest = ownedRequests[0]!; const heartbeatOwnedRequests = async () => { // Receipt claims are acquired in bulk before the provider-shaped // dispatcher admits every call. Renew the undispatched tail as well as // in-flight calls; otherwise a slow provider can let later claims // expire while they are still waiting behind the concurrency gate. const unsettled = ownedRequests.filter((request) => this.toolCallResolvers.has(request.callId), ); if (unsettled.length === 0) return 'terminal' as const; const byReceiptKey = new Map(); for (const request of unsettled) { const receiptKey = request.receiptKey!; const group = byReceiptKey.get(receiptKey) ?? []; group.push(request); byReceiptKey.set(receiptKey, group); } // One content-addressed receipt can have many same-run followers. The // store renews unique receipt rows, not request positions, so dedupe by // key while preserving the matching per-row lease id. const entries = [...byReceiptKey]; const keys = entries.map(([receiptKey]) => receiptKey); const leaseIds = entries.map( ([, requests]) => requests[0]!.receiptLeaseId!, ); const receipts = await heartbeatReceipts({ runId: this.currentReceiptOwnerRunId, runAttempt: this.currentRunAttempt, keys, leaseIds, }); const renewedExpiries: number[] = []; for (let index = 0; index < entries.length; index += 1) { const [receiptKey, requests] = entries[index]!; const leaseId = leaseIds[index]!; // Completion can race this bulk heartbeat response. A settled // receipt group no longer needs ownership and must not turn that // race into a false lease-loss failure or hold the confirmation // horizon at its old expiry. if ( this.runtimeToolReceiptCompletionsInFlight.has(receiptKey) || !requests.some((request) => this.toolCallResolvers.has(request.callId), ) ) { continue; } const receipt = receipts[index] ?? null; const renewedExpiryMs = Date.parse(receipt?.leaseExpiresAt ?? ''); if ( !this.runtimeToolReceiptStillOwned(receipt, leaseId) || !Number.isFinite(renewedExpiryMs) ) { throw new RuntimeReceiptLeaseLostError({ receiptKey, runId: this.currentReceiptOwnerRunId, leaseId, }); } renewedExpiries.push(renewedExpiryMs); } if (renewedExpiries.length === 0) return 'terminal' as const; ownershipConfirmedUntilMs = Math.min(...renewedExpiries); return 'active' as const; }; const perCallHeartbeatIntervalMs = runtimeLeaseHeartbeatIntervalFromExpiry({ leaseExpiresAt: earliestLeaseExpiry, fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS, }); const supervisor = createRuntimeReceiptHeartbeatSupervisor({ // The cohort owns claims that have not reached provider admission yet. // Renew it before the one-call supervisors can win the same short-lease // timer race, so the queued tail remains fenced while the active call // consumes the provider concurrency slot. intervalMs: Math.max(1, Math.floor(perCallHeartbeatIntervalMs / 2)), heartbeat: heartbeatOwnedRequests, isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError, onLeaseLost: (error) => { leaseLost ??= error; }, onTransientFailure: (error) => { this.log( `Pending tool receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`, ); }, }); return { stop: () => supervisor.stop(), leaseLost: () => leaseLost, confirmOwned: async () => { // The scheduler receipt gateway returns `running` from an atomic // claim. That claim is already the provider-execution fence, so short // calls only need scheduled renewal if they approach lease expiry. // Other callers may still return queued/pending claims or omit the // explicit backend capability; retain the immediate ownership check // for those receipts. if (claimsEstablishedExecutionFence) { if ( ownershipConfirmedUntilMs - Date.now() <= IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS ) { const outcome = await heartbeatOwnedRequests(); if (outcome === 'active') supervisor.start(); return; } supervisor.start(); return; } const outcome = await heartbeatOwnedRequests(); if (outcome === 'active') supervisor.start(); }, assertOwned: () => { if (leaseLost) throw leaseLost; if (Date.now() < ownershipConfirmedUntilMs) return; throw new RuntimeReceiptLeaseLostError({ receiptKey: firstOwnedRequest.receiptKey!, runId: this.currentReceiptOwnerRunId, leaseId: firstOwnedRequest.receiptLeaseId!, }); }, }; } private async executeBatchedToolCalls( queuedToolCalls: ToolCallRequest[], ): Promise { // Group by toolId const byTool = new Map(); for (const req of queuedToolCalls) { if (!byTool.has(req.toolId)) byTool.set(req.toolId, []); byTool.get(req.toolId)!.push(req); } const toolSettlements = await Promise.allSettled( [...byTool.entries()].map(async ([toolId, requests]) => { this.log(`Executing tool batch ${toolId}: ${requests.length} calls`); const successfulLiveStepCallIds = new Set(); const recordToolStep = (stepRequests: ToolCallRequest[]): void => { if (stepRequests.length === 0) return; const stepResults: PlayStepRowResult[] = stepRequests.map((req) => { const success = successfulLiveStepCallIds.has(req.callId) || this.getCachedToolResult(toolId, req.cacheKey)?.result != null; return success ? { rowId: req.rowId, status: 'completed', success: true, error: null, } : { rowId: req.rowId, status: 'failed', success: false, error: 'Tool call failed', }; }); // Step traces are lifecycle observability, not a second row/receipt // store. Keep only call ids long enough to record their bounded // status preview. for (const request of stepRequests) { successfulLiveStepCallIds.delete(request.callId); } const toolStep = { type: 'tool' as const, toolId, // Keep the step trace preview-sized for large map pages. results: compactRowResultsPreview(stepResults), description: normalizeStepDescription(stepRequests[0]?.description), }; if (this.activeDatasetStep) { this.activeDatasetStep.substeps.push(toolStep); } else { this.steps.push(toolStep); } }; const pendingRequests: ToolCallRequest[] = []; let pendingReceiptClaimsEstablishExecutionFence = this.#options.runtimeReceiptClaimsEstablishExecutionFence === true; const recoveredRequests: ToolCallRequest[] = []; for (const req of requests) { const cached = req.cacheable === false ? undefined : req.force ? undefined : this.getCachedToolResult(toolId, req.cacheKey); if (cached?.done) { this.log(` Row ${req.rowId} ${toolId}: recovered from checkpoint`); const resolver = this.toolCallResolvers.get(req.callId); if (resolver) { resolver.resolve(cached.result); this.toolCallResolvers.delete(req.callId); } recoveredRequests.push(req); } else { pendingRequests.push(req); } } recordToolStep(recoveredRequests); const liveFollowersByOwnerCallId = new Map(); const resolveLiveFollowers = ( owner: ToolCallRequest, result: unknown, ): void => { const followers = liveFollowersByOwnerCallId.get(owner.callId); if (!followers || followers.length === 0) return; liveFollowersByOwnerCallId.delete(owner.callId); for (const follower of followers) { const followerReceiptKey = follower.receiptKey?.trim() || null; const followerResult = followerReceiptKey ? markToolExecuteResultExecutionOutcome(result, { kind: 'in_flight', receiptKey: followerReceiptKey, attachedToReceiptKey: followerReceiptKey, }) : result; if (follower.cacheable !== false) { this.cacheToolResult(toolId, follower.cacheKey, followerResult); } const resolver = this.toolCallResolvers.get(follower.callId); if (resolver) { resolver.resolve(followerResult); this.toolCallResolvers.delete(follower.callId); } this.emitScopedFieldMetaUpdate({ rowId: follower.rowId, key: follower.rowKey ?? null, tableNamespace: follower.tableNamespace ?? null, fieldName: follower.fieldName, status: 'cached', rowStatus: 'running', stage: toolId, provider: null, error: null, reused: true, dataPatch: {}, }); } }; const rejectWithLiveFollowers = async ( owner: ToolCallRequest, error: unknown, ): Promise => { const followers = liveFollowersByOwnerCallId.get(owner.callId) ?? []; liveFollowersByOwnerCallId.delete(owner.callId); const shouldPersistReceiptFailure = !(error instanceof ToolExecuteAuthScopeChangedError) && !this.persistenceLatch.tripped; for (const request of [owner, ...followers]) { await this.rejectToolCall(toolId, request, error, { persistReceiptFailure: shouldPersistReceiptFailure, }); } }; const requestsWithLiveFollowers = ( owner: ToolCallRequest, ): ToolCallRequest[] => [ owner, ...(liveFollowersByOwnerCallId.get(owner.callId) ?? []), ]; const resolveRuntimeTimeoutMsForRequests = ( requests: ToolCallRequest[], ): number | undefined => { const timeoutMs = Math.max( ...requests .map((request) => request.timeoutMs) .filter( (candidate): candidate is number => typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0, ), 0, ); return timeoutMs > 0 ? timeoutMs : undefined; }; const resolveRuntimeTimeoutMsForClaimedOwners = ( owners: ToolCallRequest[], ): number | undefined => { let timeoutMs = 0; for (const owner of owners) { if ( typeof owner.timeoutMs !== 'number' || !Number.isFinite(owner.timeoutMs) || owner.timeoutMs <= 0 ) { return undefined; } timeoutMs = Math.max( timeoutMs, resolveRuntimeTimeoutMsForRequests( requestsWithLiveFollowers(owner), ) ?? 0, ); } return timeoutMs > 0 ? timeoutMs : undefined; }; const durableExistingRunningWaits: Array<{ receiptKey: string; requestsForKey: ToolCallRequest[]; forceAfterWait: boolean; waitMaxAttempts?: number; }> = []; const durableExistingRunningState: { settlement: Promise< { status: 'fulfilled' } | { status: 'rejected'; reason: unknown } > | null; } = { settlement: null }; const durableExistingRunningAbort = new AbortController(); if ( pendingRequests.length > 0 && (this.#options.claimRuntimeStepReceipts || this.#options.claimRuntimeStepReceipt) ) { await this.withRuntimeReceiptHydrationTurn(async () => { const requestsByReceiptKey = new Map< string, { requests: ToolCallRequest[]; forceRefresh: boolean; forceFailedRefresh: boolean; } >(); const localOnlyRequests: ToolCallRequest[] = []; for (const request of pendingRequests) { const receiptKey = request.receiptKey?.trim() || null; if (!receiptKey) { localOnlyRequests.push(request); continue; } const group = requestsByReceiptKey.get(receiptKey) ?? { requests: [], forceRefresh: false, forceFailedRefresh: false, }; group.requests.push(request); group.forceRefresh ||= request.force === true; group.forceFailedRefresh ||= request.forceFailedRefresh === true; requestsByReceiptKey.set(receiptKey, group); } const normalReceiptKeys = [...requestsByReceiptKey] .filter( ([, group]) => !group.forceRefresh && !group.forceFailedRefresh, ) .map(([receiptKey]) => receiptKey); const forcedReceiptKeys = [...requestsByReceiptKey] .filter(([, group]) => group.forceRefresh) .map(([receiptKey]) => receiptKey); const failedRefreshReceiptKeys = [...requestsByReceiptKey] .filter( ([, group]) => !group.forceRefresh && group.forceFailedRefresh, ) .map(([receiptKey]) => receiptKey); const forcedExisting = forcedReceiptKeys.length ? await this.getRuntimeStepReceipts(forcedReceiptKeys) : new Map(); // A forced refresh must never duplicate a provider call owned by a // live lease. It can, however, immediately claim pending/unleased // work and any expired lease. Waiting first for those receipts turns // an already-recoverable interruption into a five-minute stall. const forcedLiveReceiptKeys = new Set( [...forcedExisting] .filter(([, receipt]) => { if ( receipt.status !== 'queued' && receipt.status !== 'pending' && receipt.status !== 'running' ) { return false; } if (!receipt.leaseId?.trim()) return false; const expiresAt = receipt.leaseExpiresAt ? Date.parse(receipt.leaseExpiresAt) : Number.NaN; // Keep an unparseable leased receipt conservative. The // fenced claim remains the recovery path after the wait. return !Number.isFinite(expiresAt) || expiresAt > Date.now(); }) .map(([receiptKey]) => receiptKey), ); const claimableForcedReceiptKeys = forcedReceiptKeys.filter( (receiptKey) => !forcedLiveReceiptKeys.has(receiptKey), ); const claims = normalReceiptKeys.length > 0 ? await this.claimRuntimeStepReceipts( normalReceiptKeys, this.currentReceiptOwnerRunId, ) : new Map(); const forcedClaims = claimableForcedReceiptKeys.length > 0 ? await this.claimRuntimeStepReceipts( claimableForcedReceiptKeys, this.currentReceiptOwnerRunId, false, true, ) : new Map(); const failedRefreshClaims = failedRefreshReceiptKeys.length > 0 ? await this.claimRuntimeStepReceipts( failedRefreshReceiptKeys, this.currentReceiptOwnerRunId, false, false, true, ) : new Map(); for (const [receiptKey, claim] of forcedClaims) { claims.set(receiptKey, claim); } for (const [receiptKey, claim] of failedRefreshClaims) { claims.set(receiptKey, claim); } const claimedRequests: ToolCallRequest[] = [...localOnlyRequests]; const durableRecoveredRequests: ToolCallRequest[] = []; const resolveRequestsFromReceipt = async ( requestsForKey: ToolCallRequest[], receipt: RuntimeStepReceipt, source: 'cache' | 'in_flight', ): Promise => { const request = requestsForKey[0]; if (!request) return; const receiptKey = request.receiptKey?.trim() || null; if (!receiptKey) { throw new Error(`${source} tool recovery needs receipt key.`); } const wrapped = await this.wrapToolExecutionResult({ toolId, status: receipt.output === null || receipt.output === undefined ? 'no_result' : 'completed', result: this.runtimeReceiptOutput(receipt), requestInput: request.input, execution: toolExecutionMetadataForOutcome({ kind: source, cacheKey: request.cacheKey, receiptKey, attachedToReceiptKey: receiptKey, }), }); this.cacheToolResult(toolId, request.cacheKey, wrapped); for (const waitingRequest of requestsForKey) { successfulLiveStepCallIds.add(waitingRequest.callId); const resolver = this.toolCallResolvers.get( waitingRequest.callId, ); if (resolver) { resolver.resolve(wrapped); this.toolCallResolvers.delete(waitingRequest.callId); } this.emitScopedFieldMetaUpdate({ rowId: waitingRequest.rowId, key: waitingRequest.rowKey ?? null, tableNamespace: waitingRequest.tableNamespace ?? null, fieldName: waitingRequest.fieldName, status: 'cached', rowStatus: 'running', stage: toolId, provider: null, error: null, reused: true, dataPatch: {}, }); } }; const claimOwnerWithLiveFollowers = ( owner: ToolCallRequest | undefined, followers: ToolCallRequest[], ): void => { if (!owner) return; claimedRequests.push(owner); if (followers.length > 0) { liveFollowersByOwnerCallId.set(owner.callId, followers); } }; const resolveReceiptWaitMaxAttempts = ( requestsForKey: ToolCallRequest[], ): number | undefined => requestsForKey.length > 0 ? Math.max( ...requestsForKey.map((request) => resolveRuntimeToolReceiptWaitMaxAttempts( typeof request.receiptWaitMs === 'number' ? { max_wait_ms: request.receiptWaitMs } : request.input, ), ), ) : undefined; const queueExistingRunningReceiptWait = ( receiptKey: string, requestsForKey: ToolCallRequest[], forceAfterWait = false, ): void => { durableExistingRunningWaits.push({ receiptKey, requestsForKey, forceAfterWait, waitMaxAttempts: resolveReceiptWaitMaxAttempts(requestsForKey), }); }; const reclaimExistingRunningReceiptAfterWait = async ( receiptKey: string, requestsForKey: ToolCallRequest[], forceAfterWait = false, waitResult?: { completedReceipt?: RuntimeStepReceipt; skipWait?: boolean; }, ): Promise => { if (waitResult?.completedReceipt) { if (!forceAfterWait) { await resolveRequestsFromReceipt( requestsForKey, waitResult.completedReceipt, 'in_flight', ); return; } } else if (waitResult?.skipWait !== true) { const waitMaxAttempts = resolveReceiptWaitMaxAttempts(requestsForKey); try { const completed = await this.waitForCompletedRuntimeToolReceipt( receiptKey, waitMaxAttempts, ); if (!forceAfterWait) { await resolveRequestsFromReceipt( requestsForKey, completed, 'in_flight', ); return; } } catch (error) { if (!(error instanceof RuntimeReceiptWaitTimeoutError)) { for (const request of requestsForKey) { await this.rejectToolCall(toolId, request, error, { persistReceiptFailure: false, }); } return; } } } const reclaimed = ( await this.claimRuntimeStepReceipts( [receiptKey], this.currentReceiptOwnerRunId, true, forceAfterWait, ) ).get(receiptKey); if ( reclaimed?.status === 'completed' || reclaimed?.status === 'skipped' ) { await resolveRequestsFromReceipt( requestsForKey, reclaimed, 'cache', ); durableRecoveredRequests.push(...requestsForKey); return; } if (reclaimed?.status === 'failed') { for (const request of requestsForKey) { await this.rejectToolCall( toolId, request, runtimeReceiptFailureError( reclaimed, 'Durable tool call failed', this.currentToolErrorSchemaVersion, ), ); } return; } if (this.isOwnedClaimedRuntimeReceipt(reclaimed)) { const [owner, ...waiters] = requestsForKey; if (!owner) return; if (!reclaimed.leaseId) { throw new RuntimeReceiptLeaseLostError({ receiptKey, runId: this.currentReceiptOwnerRunId, leaseId: 'missing', }); } owner.receiptLeaseId = reclaimed.leaseId ?? null; owner.receiptLeaseExpiresAt = reclaimed.leaseExpiresAt ?? null; if (waiters.length > 0) { liveFollowersByOwnerCallId.set(owner.callId, waiters); } try { const execution = await this.callToolExecutionAPI( toolId, owner.input, { beforeProviderCall: () => this.assertRuntimeToolReceiptOwnership([owner]), durableCallReceiptKey: receiptKey, playNodeScope: playNodeScopeForToolCallRequest(owner), executionAuthScopeDigest: owner.executionAuthScopeDigest, providerIdempotencyReceiptKey: owner.providerIdempotencyKeyBase ?? owner.cacheKey, receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt, heartbeatReceipt: () => this.renewRuntimeToolReceiptOwnership([owner]), providerIdempotencyKey: this.providerIdempotencyKeyForToolCall({ cacheKey: owner.providerIdempotencyKeyBase ?? owner.cacheKey, force: owner.force === true, leaseId: owner.receiptLeaseId, }), timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([ owner, ]), }, ); const result = await this.resolveToolCall( toolId, owner, execution?.result ?? null, execution?.metadata ?? null, execution?.jobId, execution?.meta, execution?.toolResponse, ); if (result != null) { successfulLiveStepCallIds.add(owner.callId); } resolveLiveFollowers(owner, result); recordToolStep([owner]); this.#options.onBatchComplete?.(this.checkpoint); } catch (error) { await rejectWithLiveFollowers(owner, error); } return; } for (const request of requestsForKey) { await this.rejectToolCall( toolId, request, new RuntimeReceiptWaitTimeoutError(receiptKey), { persistReceiptFailure: false }, ); } }; const processExistingRunningReceiptWaits = async (): Promise => { if (durableExistingRunningWaits.length === 0) return; const waitSignal = durableExistingRunningAbort.signal; const pendingWaits = new Map( durableExistingRunningWaits.map((wait) => [ wait.receiptKey, wait, ]), ); const waitMaxAttempts = Math.max( ...durableExistingRunningWaits.map( (wait) => wait.waitMaxAttempts ?? 0, ), ) || DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS; if (runtimeReceiptReadTraceEnabled) { this.log( `[runtime-receipt-wait] phase=start requested=${pendingWaits.size} ` + `request_digests=${[...pendingWaits.keys()] .map(runtimeReceiptKeyDigest) .join(',')} max_attempts=${waitMaxAttempts}`, ); } for ( let attempt = 0; attempt < waitMaxAttempts && pendingWaits.size > 0; attempt += 1 ) { if (waitSignal.aborted) return; if (attempt > 0) { await new Promise((resolve) => { const finish = () => { clearTimeout(timeout); waitSignal.removeEventListener('abort', finish); resolve(); }; const timeout = setTimeout( finish, DURABLE_RECEIPT_WAIT_DELAY_MS, ); waitSignal.addEventListener('abort', finish, { once: true, }); }); if (waitSignal.aborted) return; } // The long backoff lives outside the hydration lane. Only a // bounded receipt read and delivery hold the lane, so stale // ownership cannot block unrelated cache misses/provider // work while terminal payloads still hydrate one group at a // time. await this.withRuntimeReceiptHydrationTurn(async () => { if (waitSignal.aborted) return; const waited = await this.waitForCompletedRuntimeToolReceipts( [...pendingWaits.keys()], 1, ); if (waitSignal.aborted) return; const settledWaits = [...pendingWaits.values()].filter( (wait) => waited.completed.has(wait.receiptKey) || waited.failed.has(wait.receiptKey), ); const deliverySettlements = await Promise.allSettled( settledWaits.map(async (wait) => { const failed = waited.failed.get(wait.receiptKey); if (failed) { for (const request of wait.requestsForKey) { await this.rejectToolCall(toolId, request, failed, { persistReceiptFailure: false, }); } return; } await reclaimExistingRunningReceiptAfterWait( wait.receiptKey, wait.requestsForKey, wait.forceAfterWait, { completedReceipt: waited.completed.get( wait.receiptKey, ), skipWait: true, }, ); }), ); for ( let index = 0; index < settledWaits.length; index += 1 ) { if (deliverySettlements[index]?.status === 'fulfilled') { pendingWaits.delete(settledWaits[index]!.receiptKey); } } if (runtimeReceiptReadTraceEnabled) { const rejected = deliverySettlements.flatMap( (settlement, index) => { if (settlement.status !== 'rejected') return []; const receiptKey = settledWaits[index]?.receiptKey; return receiptKey ? [ `${runtimeReceiptKeyDigest(receiptKey)}:${ settlement.reason instanceof Error ? settlement.reason.name : 'non_error' }`, ] : ['unknown']; }, ); this.log( `[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` + `rejected=${rejected.length} rejected_digests=${rejected.join(',')}`, ); } const rejectedDelivery = deliverySettlements.find( (settlement) => settlement.status === 'rejected', ); if (rejectedDelivery?.status === 'rejected') { throw rejectedDelivery.reason; } }); } if (waitSignal.aborted) return; const reclaimSettlements = await Promise.allSettled( [...pendingWaits.values()].map((wait) => reclaimExistingRunningReceiptAfterWait( wait.receiptKey, wait.requestsForKey, wait.forceAfterWait, { skipWait: true }, ), ), ); const rejectedReclaim = reclaimSettlements.find( (settlement) => settlement.status === 'rejected', ); if (rejectedReclaim?.status === 'rejected') { throw rejectedReclaim.reason; } }; for (const [receiptKey, group] of requestsByReceiptKey) { const requestsForKey = group.requests; const claim = claims.get(receiptKey); if (!claim) { queueExistingRunningReceiptWait( receiptKey, requestsForKey, group.forceRefresh, ); continue; } if ( claim?.status === 'completed' || claim?.status === 'skipped' ) { await resolveRequestsFromReceipt( requestsForKey, claim, 'cache', ); durableRecoveredRequests.push(...requestsForKey); continue; } if (claim?.status === 'failed') { for (const request of requestsForKey) { await this.rejectToolCall( toolId, request, runtimeReceiptFailureError( claim, 'Durable tool call failed', this.currentToolErrorSchemaVersion, ), ); } continue; } if ( (claim?.status === 'queued' || claim?.status === 'pending' || claim?.status === 'running') && claim.claimState === 'existing' ) { queueExistingRunningReceiptWait( receiptKey, requestsForKey, group.forceRefresh, ); continue; } if (this.isOwnedClaimedRuntimeReceipt(claim)) { const [owner, ...waiters] = requestsForKey; if (!claim.leaseId) { throw new RuntimeReceiptLeaseLostError({ receiptKey, runId: this.currentReceiptOwnerRunId, leaseId: 'missing', }); } if (owner) { owner.receiptLeaseId = claim.leaseId ?? null; owner.receiptLeaseExpiresAt = claim.leaseExpiresAt ?? null; const claimLeaseExpiresAtMs = Date.parse( claim.leaseExpiresAt ?? '', ); pendingReceiptClaimsEstablishExecutionFence &&= claim.claimState === 'claimed' && claim.status === 'running' && Number.isFinite(claimLeaseExpiresAtMs) && claimLeaseExpiresAtMs > Date.now(); } claimOwnerWithLiveFollowers(owner, waiters); continue; } queueExistingRunningReceiptWait( receiptKey, requestsForKey, group.forceRefresh, ); } pendingRequests.length = 0; pendingRequests.push(...claimedRequests); durableExistingRunningState.settlement = durableExistingRunningWaits.length > 0 ? processExistingRunningReceiptWaits().then( () => ({ status: 'fulfilled' as const }), (reason: unknown) => ({ status: 'rejected' as const, reason, }), ) : null; recordToolStep(durableRecoveredRequests); }); } // Claims establish one lease cohort. Revalidate that cohort in one // gateway operation before provider admission, then let the existing // cohort heartbeat keep the undispatched tail alive. Revalidating each // call separately after a tool slot opens fragments the writer into // one-row batches and starves otherwise healthy providers. const pendingReceiptHeartbeat = this.startPendingToolReceiptHeartbeat( pendingRequests, pendingReceiptClaimsEstablishExecutionFence, ); let providerFailure: { reason: unknown } | null = null; try { if (pendingReceiptHeartbeat) { await pendingReceiptHeartbeat.confirmOwned(); } if (pendingRequests.length > 0) { const strategy = this.#options.getBatchOperationStrategy?.(toolId) ?? null; if (strategy) { const compiledBatches = compileRequestsWithStrategy({ requests: pendingRequests, strategy, getPayload: (request: ToolCallRequest) => request.input, }); const batchParallelismCeiling = this.governor.policy.pacing.workerToolBatchDefaultParallelism; const batchSize = compiledBatches.length > 0 && !this.fixtureProviderPacingDisabled() ? await this.resourceGovernor.suggestedToolParallelism( compiledBatches[0]!.batchOperation, batchParallelismCeiling, ) : batchParallelismCeiling; await executeChunkedRequests({ requests: compiledBatches, batchSize, execute: async (batch) => { // Circuit breaker: skip dispatching this batch's provider call // once a persistence failure has occurred in this run. if (this.persistenceLatch.tripped) { this.persistenceLatch.preventedCallCount += batch.memberRequests.length; throw new RuntimePersistenceCircuitOpenError( this.persistenceLatch, ); } const receiptKeys = batch.memberRequests.map( (request) => request.cacheKey, ); const providerIdempotencyReceiptKeys = batch.memberRequests.map( (request) => request.providerIdempotencyKeyBase ?? request.cacheKey, ); const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({ receiptKeys, prefix: 'batch', aggregateReceiptPrefix: buildDurableToolReceiptPrefix({ orgId: this.#options.orgId, toolId: batch.batchOperation, }), }); const aggregateProviderIdempotencyReceiptKey = buildDurableToolAggregateReceiptKey({ receiptKeys: providerIdempotencyReceiptKeys, prefix: 'batch', aggregateReceiptPrefix: buildDurableToolReceiptPrefix({ orgId: this.#options.orgId, toolId: batch.batchOperation, }), }); let releaseToolSlot: () => void = () => undefined; try { const execution = await this.callToolExecutionAPI( batch.batchOperation, batch.batchPayload, { durableCallReceiptKey: aggregateReceiptKey, // A native batch is one physical provider request for // many rows of the same column, so the batch's node scope // is its members' shared scope. playNodeScope: batch.memberRequests[0] ? playNodeScopeForToolCallRequest( batch.memberRequests[0], batch.batchOperation, ) : null, executionAuthScopeDigest: batch.memberRequests[0]?.executionAuthScopeDigest ?? null, providerIdempotencyReceiptKey: aggregateProviderIdempotencyReceiptKey, providerIdempotencyKey: buildDurableToolAggregateProviderIdempotencyKey({ aggregateReceiptKey: aggregateProviderIdempotencyReceiptKey, receiptKeys: providerIdempotencyReceiptKeys, providerIdempotencyKeys: batch.memberRequests.map( (request) => this.providerIdempotencyKeyForToolCall({ cacheKey: request.providerIdempotencyKeyBase ?? request.cacheKey, force: request.force === true, leaseId: request.receiptLeaseId, }), ), }), receiptLeaseExpiresAt: batch.memberRequests.reduce< string | null >((earliest, request) => { const expiresAt = request.receiptLeaseExpiresAt ?? null; if (!expiresAt) return earliest; if (!earliest) return expiresAt; return Date.parse(expiresAt) < Date.parse(earliest) ? expiresAt : earliest; }, null), timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners( batch.memberRequests, ), beforeProviderCall: () => pendingReceiptHeartbeat ? pendingReceiptHeartbeat.assertOwned() : this.assertRuntimeToolReceiptOwnership( batch.memberRequests, ), heartbeatReceipt: () => this.renewRuntimeToolReceiptOwnership( batch.memberRequests, ), retainToolSlot: (release) => { releaseToolSlot = release; }, }, ); return { execution, releaseToolSlot }; } catch (error) { releaseToolSlot(); throw error; } }, onChunkComplete: async (chunkResults) => { for (const entry of chunkResults) { if (entry.error !== undefined) { for (const request of entry.request.memberRequests) { await rejectWithLiveFollowers(request, entry.error); } continue; } const batchExecution = entry.result; try { const splitResults = batchExecution != null ? entry.request.splitResults( legacyResultForBatchSplitter( batchExecution.execution, ), ) : entry.request.memberRequests.map(() => null); const resolvedResults = await this.resolveToolCallBatchResults( toolId, entry.request.memberRequests.map( (request, index) => ({ request, result: splitResults[index] ?? null, status: batchExecution?.execution.status, toolResponse: batchExecution == null ? undefined : publicToolResponseForBatchedItem( batchExecution.execution, splitResults[index] ?? null, this.currentToolResponseContract === RAW_V2_TOOL_RESPONSE_CONTRACT, ), }), ), ); for ( let index = 0; index < entry.request.memberRequests.length; index += 1 ) { const request = entry.request.memberRequests[index]!; if (resolvedResults[index] != null) { successfulLiveStepCallIds.add(request.callId); } resolveLiveFollowers(request, resolvedResults[index]); } } finally { batchExecution?.releaseToolSlot(); } } recordToolStep( chunkResults.flatMap( (entry) => entry.request.memberRequests, ), ); this.#options.onBatchComplete?.(this.checkpoint); }, retainResults: false, }); } else { const completionBuffer: Array<{ request: ToolCallRequest; result: unknown | null; status?: string; metadata?: ToolResultMetadataInput | null; jobId?: string; meta?: Record; toolResponse?: ParsedToolExecuteResponse['toolResponse']; resolve: (value: unknown) => void; reject: (error: unknown) => void; }> = []; const completionFailedCallIds = new Set(); let completionFlushScheduled = false; const flushCompletionBuffer = async (): Promise => { const entries = completionBuffer.splice(0); if (entries.length === 0) return; try { const resolvedResults = await this.resolveToolCallBatchResults( toolId, entries.map((entry) => ({ request: entry.request, result: entry.result, status: entry.status, metadata: entry.metadata, jobId: entry.jobId, meta: entry.meta, toolResponse: entry.toolResponse, })), ); for (let index = 0; index < entries.length; index += 1) { const entry = entries[index]!; const result = resolvedResults[index]; if (result != null) { successfulLiveStepCallIds.add(entry.request.callId); } resolveLiveFollowers(entry.request, result); entry.resolve(result); } } catch (error) { for (const entry of entries) { completionFailedCallIds.add(entry.request.callId); await this.rejectToolCall(toolId, entry.request, error, { persistReceiptFailure: false, }); entry.reject(error); } } }; const enqueueCompletion = ( request: ToolCallRequest, execution: ParsedToolExecuteResponse, ): Promise => new Promise((resolve, reject) => { completionBuffer.push({ request, result: execution.result ?? null, status: execution.status, metadata: execution.metadata ?? null, jobId: execution.jobId, meta: execution.meta, toolResponse: execution.toolResponse, resolve, reject, }); if (completionFlushScheduled) return; completionFlushScheduled = true; setTimeout(() => { completionFlushScheduled = false; void flushCompletionBuffer(); }, 0); }); // Seed the dispatch width from the governor's provider-shaped // parallelism instead of the flat policy.concurrency.toolCalls // ceiling. The flat width launched every pending row at once into // the pacer, so unhinted providers 429'd on the first burst before // AIMD could halve the rate. The shaped estimate is derived from the // provider's RPS/maxConcurrency pacing rules (see // suggestedParallelism in governor.ts), floored at 1 and capped by // the global tool-call concurrency ceiling. The pacer still gates // each in-flight call, so this only trims the launch burst. const toolCallConcurrencyCeiling = this.governor.policy.concurrency.toolCalls; const shapedToolParallelism = this.fixtureProviderPacingDisabled() ? toolCallConcurrencyCeiling : await this.resourceGovernor.suggestedToolParallelism( toolId, toolCallConcurrencyCeiling, ); const dispatchWidth = Math.min( toolCallConcurrencyCeiling, Math.max(1, shapedToolParallelism), ); const dispatchResults = await dispatchBoundedSettled( pendingRequests, dispatchWidth, async (request) => { // Circuit breaker: do not dispatch this provider call once a // persistence failure has occurred in this run. if (this.persistenceLatch.tripped) { this.persistenceLatch.preventedCallCount += 1; throw new RuntimePersistenceCircuitOpenError( this.persistenceLatch, ); } let releaseToolSlot: () => void = () => undefined; try { const execution = await this.callToolExecutionAPI( toolId, request.input, { beforeProviderCall: () => pendingReceiptHeartbeat ? pendingReceiptHeartbeat.assertOwned() : this.assertRuntimeToolReceiptOwnership([request]), playNodeScope: playNodeScopeForToolCallRequest(request), ...(request.receiptKey ? { durableCallReceiptKey: request.receiptKey, executionAuthScopeDigest: request.executionAuthScopeDigest, providerIdempotencyReceiptKey: request.providerIdempotencyKeyBase ?? request.receiptKey, providerIdempotencyKey: this.providerIdempotencyKeyForToolCall({ cacheKey: request.providerIdempotencyKeyBase ?? request.receiptKey, force: request.force === true, leaseId: request.receiptLeaseId, }), receiptLeaseExpiresAt: request.receiptLeaseExpiresAt, heartbeatReceipt: () => this.renewRuntimeToolReceiptOwnership([ request, ]), } : {}), timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([ request, ]), retainToolSlot: (release) => { releaseToolSlot = release; }, }, ); await enqueueCompletion(request, execution); } finally { releaseToolSlot(); } }, ); const failedEntries = dispatchResults.filter( (entry) => entry.error !== undefined && !completionFailedCallIds.has(entry.job.callId), ); await Promise.allSettled( failedEntries.map((entry) => rejectWithLiveFollowers(entry.job, entry.error), ), ); recordToolStep(pendingRequests); this.#options.onBatchComplete?.(this.checkpoint); } } } catch (reason) { providerFailure = { reason }; durableExistingRunningAbort.abort(reason); } finally { pendingReceiptHeartbeat?.stop(); } const pendingReceiptLeaseLost = pendingReceiptHeartbeat?.leaseLost(); if (pendingReceiptLeaseLost && providerFailure === null) { providerFailure = { reason: pendingReceiptLeaseLost }; durableExistingRunningAbort.abort(pendingReceiptLeaseLost); } const durableExistingRunningSettlement = durableExistingRunningState.settlement; if (durableExistingRunningSettlement) { const settlement = await durableExistingRunningSettlement; if (settlement.status === 'rejected' && providerFailure === null) { providerFailure = { reason: settlement.reason }; } } if (providerFailure) throw providerFailure.reason; }), ); const rejected = toolSettlements.find( (settlement): settlement is PromiseRejectedResult => settlement.status === 'rejected', ); if (rejected) throw rejected.reason; } private async executeResolvedPlay( resolvedPlay: ResolvedPlayExecution, ctx: PlayContextImpl, input: Record, ): Promise { if (resolvedPlay.definition) { if (!this.#options.executeStructuredPlayDefinition) { throw new Error( `Play "${resolvedPlay.playId}" is a structured play, but this runtime did not provide a structured play executor.`, ); } return await this.#options.executeStructuredPlayDefinition({ definition: resolvedPlay.definition, ctx, rows: [], playInput: input, }); } if (resolvedPlay.codeFormat === 'cjs_module') { const artifact = resolvedPlay.artifact; if (!artifact) { throw new Error( `Play "${resolvedPlay.playId}" is missing a bundled artifact.`, ); } const cacheKey = resolvedPlayRevisionFingerprint(resolvedPlay); let executor = this.resolvedPlayExecutorCache.get(cacheKey); if (!executor) { executor = (async (): Promise => { const runtimeModule = (await import('node:module')) as unknown as typeof import('node:module') & { Module: typeof import('node:module').Module & { _nodeModulePaths: (from: string) => string[]; }; }; const compiled = new runtimeModule.Module(artifact.virtualFilename); compiled.filename = artifact.virtualFilename; compiled.paths = runtimeModule.Module._nodeModulePaths(process.cwd()); ( compiled as import('node:module').Module & { _compile: (code: string, filename: string) => void; } )._compile(artifact.bundledCode, artifact.virtualFilename); const candidate = typeof compiled.exports === 'function' ? compiled.exports : (compiled.exports as { default?: unknown }).default; if (typeof candidate !== 'function') { throw new Error( `Play "${resolvedPlay.playId}" does not export a callable default.`, ); } return async (childCtx, runtimeInput) => await ( candidate as ( runtimeCtx: unknown, value: Record, ) => Promise )(childCtx, runtimeInput); })(); this.resolvedPlayExecutorCache.set(cacheKey, executor); } return await ( await executor )(ctx, input); } const code = resolvedPlay.code ?? resolvedPlay.sourceCode; if (!code?.trim()) { throw new Error( `Play "${resolvedPlay.playId}" is missing executable source.`, ); } return await new Function( 'ctx', 'input', ` const __playFn = ${code}; return __playFn(ctx, input); `, )(ctx, input); } private providerIdempotencyKeyForToolCall(input: { cacheKey: string; force?: boolean; leaseId?: string | null; logicalCallId?: string; }): string { let fallbackAttemptId: string | null = null; if (input.force === true && !input.leaseId) { if (!input.logicalCallId) { throw new Error( 'Forced tool execution without a receipt lease requires a stable logical call identity.', ); } fallbackAttemptId = `${this.currentRunId}:${input.logicalCallId}`; } return buildDurableToolProviderIdempotencyKey({ receiptKey: input.cacheKey, force: input.force, receiptLeaseId: input.leaseId, fallbackAttemptId, }); } private async callToolExecutionAPI( toolId: string, input: Record, options?: ToolExecutionApiOptions, ): Promise { if (!this.#options.executorToken || !this.#options.baseUrl) { throw new Error( 'executorToken and baseUrl are required for tool API calls (cloud execution only)', ); } const requestsDurableInvocationFence = this.#options.requestDurableInvocationFence === true || this.#options.durableInvocationFence === true; const executeSuffix = requestsDurableInvocationFence ? 'execute-fenced-v1' : 'execute'; // Keep receipts, runner heartbeats, terminals, sheets, and runtime control // on baseUrl (the receipt gateway). Only the potentially long-lived // provider request may use the separately managed execution relay. const executionBaseUrl = this.#options.executionGatewayBaseUrl?.trim() || this.#options.baseUrl; const url = `${executionBaseUrl.replace(/\/$/, '')}/api/v2/integrations/${encodeURIComponent(toolId)}/${executeSuffix}`; const toolErrorSchemaVersion = this.currentToolErrorSchemaVersion; const timeoutMs = resolveToolRuntimeTimeoutMs( toolId, options?.timeoutMs, this.currentAuthoringContractEdition, ); const provider = toolId.split(/[._]/)[0]?.trim() || 'provider'; const activityId = `provider:${toolId}`; let retryActivityEmitted = false; let toolCallSucceeded = false; // Hold one logical tool slot and charge one logical tool call for this // execution. Per-provider rate admission is intentionally deferred until // each physical fetch is ready to leave this process. const admissionStartedAt = Date.now(); const toolSlot = await this.resourceGovernor .acquireTool({ orgId: this.#options.orgId ?? null, providerResourceKey: `tool:${toolId}`, toolId, }) .catch((error: unknown) => { if (!(error instanceof ProviderExhaustedError)) throw error; const retryAtMs = Date.parse(error.retryAt); throw createToolHttpError( toolErrorSchemaVersion, error.message, null, 429, 'repairable', { toolId, provider: error.provider, operation: toolId, code: error.code, origin: 'provider', category: 'rate_limit', retryable: true, statusCode: 429, requestId: null, retryAfterMs: Number.isFinite(retryAtMs) ? Math.max(0, retryAtMs - Date.now()) : null, networkKind: null, networkScope: null, }, ); }); if (runtimeReceiptReadTraceEnabled) { this.log( `[perf] tool call id=${toolId} phase=governor_admission elapsed_ms=${Date.now() - admissionStartedAt}`, ); } let toolSlotTransferred = false; try { if (options?.retainToolSlot) { options.retainToolSlot(() => toolSlot.release()); toolSlotTransferred = true; } return await withActiveSpan( 'plays.tool.execute', { tracer: 'deepline.plays', attributes: { 'plays.play_name': this.#options.playId ?? 'anonymous-play', 'plays.workflow_id': this.#options.workflowId ?? '', 'plays.run_id': this.#options.runId ?? this.#options.workflowId ?? '', 'plays.tool_id': toolId, }, }, async (span) => { const httpFailureAttempts = createToolExecuteHttpFailureAttemptTracker(); // Snapshot the caller-controlled payload exactly once. The replay // decision and every physical attempt must refer to the same bytes, // even if the original object has getters or is later mutated. const toolInputSnapshot = JSON.parse(JSON.stringify(input)) as Record< string, unknown >; const retryPolicy = await this.#options .getToolRetryPolicy?.(toolId, toolInputSnapshot) .catch(() => null); const retrySafeTransientHttp = retryPolicy?.retrySafeTransientHttp === true; const invocationOrgId = this.#options.orgId?.trim() ?? ''; const durableCallReceiptKey = options?.durableCallReceiptKey?.trim() || (requestsDurableInvocationFence && invocationOrgId ? `${buildDurableToolReceiptPrefix({ orgId: invocationOrgId, toolId, })}${stableDigest( `transport:${this.currentRunId}:${toolId}:${crypto.randomUUID()}`, )}` : null); const executionAuthScopeDigest = options?.executionAuthScopeDigest ? options.executionAuthScopeDigest.trim() || null : ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null); const providerIdempotencyKey = options?.providerIdempotencyKey?.trim() || durableCallReceiptKey; const providerIdempotencyReceiptKey = options?.providerIdempotencyReceiptKey?.trim() || durableCallReceiptKey; // Correlation identity is stable across every transport retry, // including calls without a durable receipt. const deeplineRequestId = providerIdempotencyKey ? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}` : `ctx-tool-${crypto.randomUUID()}`; // Every runtime provider call carries at least a tool-scoped // attribution. Callers that know the execution location (row-scoped // map cells, batched owners) supply the fuller scope. const playNodeScope = options?.playNodeScope ?? buildPlayNodeScope({ toolId }); const serializedRequestBody = (invocationAttempt: number) => JSON.stringify({ payload: toolInputSnapshot, metadata: { parent_run_id: this.#options.runId, invocation_attempt: invocationAttempt, ...(requestsDurableInvocationFence ? { invocation_fence_version: 1 } : {}), // This is deliberately the provider receipt, rather than the // worker-owned response cache receipt. It keeps a newer // worker wire-compatible with an already active Vercel app // while its runtime cache can still partition by response // representation. ...(providerIdempotencyReceiptKey ? { durable_call_receipt_key: providerIdempotencyReceiptKey, ...(executionAuthScopeDigest ? { execution_auth_scope_digest: executionAuthScopeDigest, } : {}), } : {}), ...(options?.customerDbDataset ? { query_result_dataset: { limit: options.customerDbDataset.limit, offset: options.customerDbDataset.offset, page_size: options.customerDbDataset.pageSize, total_rows: options.customerDbDataset.totalRows, }, ...(isCustomerDbDatasetTool(toolId) ? { customer_db_dataset: { limit: options.customerDbDataset.limit, offset: options.customerDbDataset.offset, page_size: options.customerDbDataset.pageSize, total_rows: options.customerDbDataset.totalRows, }, } : {}), } : {}), ...(providerIdempotencyKey ? { provider_idempotency_key: providerIdempotencyKey } : {}), ...(playNodeScope ? { play_node_scope: playNodeScopeToWire(playNodeScope) } : {}), }, ...(this.#options.integrationMode ? { integration_mode: this.#options.integrationMode } : {}), }); let transportAttempt = 0; let invocationAttempt = 0; let physicalAttempt = 0; const retryToolTransportFailure = async (input: { error: unknown; elapsedMs: number; requestId: string | null; aborted: boolean; }): Promise => { transportAttempt += 1; const diagnostic = describeTransportError(input.error); this.log( `[runtime.transport_failure] ${JSON.stringify({ tool_id: toolId, gateway_origin: transportGatewayOriginForDiagnostic(url), attempt: transportAttempt, max_attempts: TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS, elapsed_ms: input.elapsedMs, request_id: input.requestId, aborted: input.aborted, error: diagnostic, })}`, ); if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) { await this.governor.chargeBudget('retry'); const retryAfterMs = TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt; span.setAttribute( 'plays.transport_retry_attempt', transportAttempt, ); this.log( `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${diagnostic.message ?? 'unknown transport error'}`, ); await options?.parkProviderCall?.(); await this.sleepWithCheckpointHeartbeat(retryAfterMs); return; } throw createToolHttpError( toolErrorSchemaVersion, `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${diagnostic.message ?? 'unknown transport error'}`, null, 0, 'repairable', { toolId, provider, operation: toolId, code: 'NETWORK_ERROR', origin: 'deepline', category: 'network', retryable: false, statusCode: null, requestId: input.requestId, retryAfterMs: null, networkKind: diagnostic.message ?.toLowerCase() .includes('timed out') ? 'timeout' : 'unavailable', networkScope: 'runtime_to_deepline', }, ); }; while (true) { physicalAttempt += 1; let response: Response | null = null; let responseData: Record | null = null; let responseErrorText: string | null = null; let providerCallStartedAt: number | null = null; let providerCallElapsedMs: number | null = null; let integrationFetchStartedAt: number | null = null; let fetchDispatched = false; // Receipt ownership is a liveness contract, not a one-time check. // Keep it alive for the whole provider HTTP request. The cadence // comes from the store-issued expiry because a remote runner may // not share the coordinator's TTL environment. const hasReceiptHeartbeat = Boolean(options?.heartbeatReceipt); const abortController = timeoutMs || hasReceiptHeartbeat ? new AbortController() : null; const receiptHeartbeat = options?.heartbeatReceipt; let heartbeatFailure: unknown = null; let timeoutHandle: ReturnType | null = null; try { const ownershipStartedAt = Date.now(); await options?.beforeProviderCall?.(); if (runtimeReceiptReadTraceEnabled) { this.log( `[perf] tool call id=${toolId} phase=before_provider_ownership elapsed_ms=${Date.now() - ownershipStartedAt}`, ); } const heartbeatIntervalMs = hasReceiptHeartbeat ? runtimeLeaseHeartbeatIntervalFromExpiry({ leaseExpiresAt: options?.receiptLeaseExpiresAt, fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS, }) : null; const receiptHeartbeatSupervisor = heartbeatIntervalMs !== null && receiptHeartbeat ? createRuntimeReceiptHeartbeatSupervisor({ intervalMs: heartbeatIntervalMs, heartbeat: async () => { await receiptHeartbeat(); return 'active'; }, isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError, onLeaseLost: (error) => { heartbeatFailure ??= error; abortController?.abort(error); }, onTransientFailure: (error) => { this.log( `Tool ${toolId} receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`, ); }, }) : null; receiptHeartbeatSupervisor?.start(); try { // Complete all local/receipt preparation before taking the // shared provider permit. Once admitted, the next awaited work // is the fetch itself, so independently delayed runners cannot // compress real provider arrivals after their tickets. const protectionHeaders = await this.vercelProtectionHeaders(); // Use the mode carried by this physical execution request. // A restored execution scope may predate integration-mode // authority metadata, while the signed launch and every tool // request still carry options.integrationMode. Pacing the // latter as live would make fixture runs both slow and // unauditable even though the app never dispatches a provider // request. const fixtureExecution = this.#options.integrationMode === 'fixture'; const enforceFixtureProviderPacing = fixtureExecution && this.#options.enforceFixtureProviderPacing === true; const fixtureOnlyExecution = this.fixtureProviderPacingDisabled(); if ( fixtureOnlyExecution && !this.fixtureProviderPacingBypassLogged ) { this.fixtureProviderPacingBypassLogged = true; this.log( 'Fixture mode: provider pacing bypassed because no provider request is dispatched.', ); } if ( enforceFixtureProviderPacing && !this.fixtureProviderPacingEnforcementLogged ) { this.fixtureProviderPacingEnforcementLogged = true; this.log( 'Fixture mode: provider pacing explicitly enforced for production-parity testing.', ); } const providerPermit = fixtureOnlyExecution ? { release() {} } : await this.resourceGovernor.acquireProviderPermit({ toolId, signal: abortController?.signal, }); try { // Provider admission is our queue, not provider execution. // Start the tool deadline only after admission so a busy // runtime cannot consume the remote-call budget while this // attempt is still waiting for permission to leave. if ( timeoutMs && abortController && !abortController.signal.aborted ) { timeoutHandle = setTimeout(() => { abortController.abort( new Error( `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`, ), ); }, timeoutMs); } const fixtureBehavior = this.#options.fixtureBehavior; if ( fixtureExecution && fixtureBehavior && shouldRouteFixtureToolId(toolId) ) { const canonicalProvider = (await this.#options.getToolProvider?.(toolId))?.trim() || provider; if (shouldRouteFixtureProvider(canonicalProvider)) { if (!durableCallReceiptKey) { throw new Error( 'Configured fixture response timing requires a durable call receipt key.', ); } const canonicalOperation = ( await this.#options.getToolOperation?.(toolId) )?.trim() || toolId; const stableRequestKey = `${canonicalProvider}:${canonicalOperation}:${durableCallReceiptKey}`; const behaviorDigest = stableDigest( JSON.stringify(fixtureBehavior), ); const requestKeyDigest = stableDigest(stableRequestKey); let selected: { delayMs: number; sampleIndex: number; } | null = null; const delayStartedAt = Date.now(); providerCallStartedAt = delayStartedAt; try { selected = await waitForFixtureResponseDelay({ behavior: fixtureBehavior, stableRequestKey, signal: abortController?.signal, onSelected: (value) => { selected = value; this.log( `[fixture.response_delay] ${JSON.stringify({ outcome: 'scheduled', provider: canonicalProvider, operation: canonicalOperation, behavior_digest: `sha256_${behaviorDigest}`, request_key_digest: `sha256_${requestKeyDigest}`, delay_ms: value.delayMs, sample_index: value.sampleIndex, physical_attempt: physicalAttempt, })}`, ); }, }); } catch (error) { this.log( `[fixture.response_delay] ${JSON.stringify({ outcome: 'aborted', provider: canonicalProvider, operation: canonicalOperation, behavior_digest: `sha256_${behaviorDigest}`, request_key_digest: `sha256_${requestKeyDigest}`, delay_ms: selected?.delayMs ?? null, sample_index: selected?.sampleIndex ?? null, physical_attempt: physicalAttempt, elapsed_ms: Date.now() - delayStartedAt, })}`, ); throw error; } this.log( `[fixture.response_delay] ${JSON.stringify({ outcome: 'completed', provider: canonicalProvider, operation: canonicalOperation, behavior_digest: `sha256_${behaviorDigest}`, request_key_digest: `sha256_${requestKeyDigest}`, delay_ms: selected.delayMs, sample_index: selected.sampleIndex, physical_attempt: physicalAttempt, elapsed_ms: Date.now() - delayStartedAt, })}`, ); } } const integrationRequestLease = await this.governor.acquireIntegrationRequestSlot({ signal: abortController?.signal, }); try { integrationFetchStartedAt = Date.now(); providerCallStartedAt ??= integrationFetchStartedAt; fetchDispatched = true; response = await fetch(url, { method: 'POST', signal: abortController?.signal, headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${this.#options.executorToken}`, [EXECUTE_RESPONSE_CONTRACT_HEADER]: this.currentToolResponseContract, [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset', [EXECUTE_TOOL_METADATA_HEADER]: 'true', [TOOL_EXECUTION_ERROR_SCHEMA_HEADER]: String( toolErrorSchemaVersion, ), 'x-deepline-request-id': deeplineRequestId, ...(providerIdempotencyKey ? { 'x-deepline-idempotency-key': providerIdempotencyKey, } : {}), ...protectionHeaders, ...(this.#options.runtimeTestFaultHeader ? { 'x-deepline-test-fault': this.#options.runtimeTestFaultHeader, } : {}), }, body: serializedRequestBody(invocationAttempt), }); if (response.ok) { try { responseData = await readToolExecuteResponseBody({ toolId, abortController, read: () => response!.json() as Promise< Record >, }); } catch (error) { if (error instanceof SyntaxError) { throw new ToolExecuteInvalidJsonError(error); } throw new ToolExecuteResponseBodyTransportError(error); } } else { try { responseErrorText = await readToolExecuteResponseBody({ toolId, abortController, read: () => response!.text(), }); } catch (error) { throw new ToolExecuteResponseBodyTransportError(error); } } } finally { integrationRequestLease.release(); } } finally { providerPermit.release(); } if ( runtimeReceiptReadTraceEnabled && integrationFetchStartedAt !== null ) { this.log( `[perf] tool call id=${toolId} phase=integration_fetch_body elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`, ); } providerCallElapsedMs = Date.now() - providerCallStartedAt; this.resourceGovernor.observe({ toolId, providerResourceKey: `tool:${toolId}`, providerLatencyMs: providerCallElapsedMs, providerSuccess: response.ok, provider429: response.status === 429, }); } finally { receiptHeartbeatSupervisor?.stop(); } } catch (error) { if (heartbeatFailure) { throw heartbeatFailure; } const transportError = abortController?.signal.aborted ? abortController.signal.reason instanceof Error ? abortController.signal.reason : new Error( `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`, ) : error instanceof ToolExecuteResponseBodyTransportError || error instanceof ToolExecuteInvalidJsonError ? error.cause : error; if ( (error instanceof ToolExecuteResponseBodyTransportError || error instanceof ToolExecuteInvalidJsonError) && response?.status === 402 ) { const diagnostic = describeTransportError(transportError); throw createToolHttpError( toolErrorSchemaVersion, `Tool ${toolId} returned HTTP 402 but its response body could not be read; the run was halted because the payment or capacity denial could not be classified safely: ${diagnostic.message ?? 'unknown transport error'}`, { kind: 'billing_cap_exceeded', code: 'HTTP_402_BODY_UNREADABLE', error_category: 'billing', failure_origin: 'unknown', }, 402, ); } const ambiguousDispatchedFailure = fetchDispatched && (response === null || error instanceof ToolExecuteResponseBodyTransportError || error instanceof ToolExecuteInvalidJsonError); const hasDurableInvocationIdentity = Boolean( durableCallReceiptKey && providerIdempotencyKey && executionAuthScopeDigest, ); // A post-header body deadline aborts the fetch controller solely // to stop consuming that response. It must not also cancel the // independent gateway health check that establishes whether the // committed response can be replayed safely. Preserve genuine // caller/runtime cancellation by dropping the signal only when // this exact body deadline caused the abort. const durableFenceVerificationSignal = abortController?.signal.aborted === true && abortController.signal.reason instanceof ToolExecuteResponseBodyTimeoutError ? undefined : abortController?.signal; const durableInvocationFenceVerified = hasDurableInvocationIdentity && requestsDurableInvocationFence && (this.#options.durableInvocationFence === true || (await this.#options .verifyDurableInvocationFence?.( durableFenceVerificationSignal, ) .catch(() => false)) === true); // Fixture execution has no provider-side effect. Replaying the // same stable request after our own transport fails is always // safe, even when this runner lacks a live-provider fence. const fixtureReplaySafe = this.#options.integrationMode === 'fixture'; const transportReplaySafe = !ambiguousDispatchedFailure || durableInvocationFenceVerified || fixtureReplaySafe; if (!transportReplaySafe) { const diagnostic = describeTransportError(transportError); this.log( `[runtime.transport_failure] ${JSON.stringify({ tool_id: toolId, gateway_origin: transportGatewayOriginForDiagnostic(url), attempt: 1, max_attempts: 1, elapsed_ms: providerCallStartedAt === null ? 0 : Date.now() - providerCallStartedAt, request_id: deeplineRequestId, aborted: abortController?.signal.aborted === true, response_headers_received: response !== null, retry_safe: false, error: diagnostic, })}`, ); const failureBoundary = error instanceof ToolExecuteResponseBodyTransportError || error instanceof ToolExecuteInvalidJsonError ? 'response body could not be verified after response headers' : 'request transport failed after dispatch before response headers'; throw createToolHttpError( toolErrorSchemaVersion, `Tool ${toolId} ${failureBoundary}; the ambiguous call was not retried because this runtime has no durable invocation fence: ${diagnostic.message ?? 'unknown transport error'}`, null, 0, 'repairable', ); } if ( error instanceof ToolExecuteResponseBodyTransportError && response?.status === 429 ) { await this.reportToolBackpressure( toolId, parseToolExecuteRetryAfterMs( response.headers.get('retry-after'), ), ); } await retryToolTransportFailure({ error: transportError, elapsedMs: providerCallStartedAt === null ? 0 : Date.now() - providerCallStartedAt, requestId: deeplineRequestId, aborted: abortController?.signal.aborted === true, }); continue; } finally { if (timeoutHandle) { clearTimeout(timeoutHandle); } } if (!response) { throw new Error( `Tool ${toolId} transport completed without an HTTP response.`, ); } span.setAttribute('plays.http_status_code', response.status); if (!response.ok) { const text = responseErrorText ?? ''; const developerTunnelOrigin502 = isDeeplineDeveloperTunnelOrigin502({ url, status: response.status, bodyText: text, }); const unmarkedExecutionGateway502 = isUnmarkedExecutionGateway502( { url, status: response.status, responseHeaders: response.headers, }, ); if (developerTunnelOrigin502 || unmarkedExecutionGateway502) { await retryToolTransportFailure({ error: new Error( developerTunnelOrigin502 ? 'the Deepline development tunnel returned a branded 502 before reaching the app origin' : 'the execution gateway ingress returned an unmarked 502 before reaching the durable invocation handler', ), elapsedMs: providerCallElapsedMs ?? 0, requestId: deeplineRequestId, aborted: false, }); continue; } const authScopeChanged = parseToolExecuteAuthScopeChangedError({ status: response.status, bodyText: text, }); if (authScopeChanged) { throw authScopeChanged; } const httpFailureAttempt = httpFailureAttempts.next({ toolId, status: response.status, bodyText: text, transientHttpRetrySafe: retrySafeTransientHttp, }); const failure = classifyToolExecuteHttpFailure({ toolId, status: response.status, attempt: httpFailureAttempt, bodyText: text, schemaVersion: toolErrorSchemaVersion, retryAfterHeader: response.headers.get('retry-after'), transientHttpRetrySafe: retrySafeTransientHttp, ...(providerCallElapsedMs !== null ? { providerLatencyMs: providerCallElapsedMs } : {}), }); if (failure.backpressureDelayMs !== null) { // Feed the server-observed Retry-After back into the shared // pacer even on the final attempt so later provider calls back // off instead of retrying the whole map chunk. await this.reportToolBackpressure( toolId, failure.backpressureDelayMs, ); } if (failure.shouldRetry) { if (failure.reason !== 'gateway_invocation_in_progress') { invocationAttempt += 1; } if (failure.chargeRetryBudget) { await this.governor.chargeBudget('retry'); } const retryAttributePrefix = failure.isRateLimit ? 'rate_limit' : 'transient_http'; span.setAttribute( `plays.${retryAttributePrefix}_retry_after_ms`, failure.retryDelayMs, ); span.setAttribute( `plays.${retryAttributePrefix}_attempt`, httpFailureAttempt, ); this.log( `Tool ${toolId} returned ${response.status}; retrying after ${failure.retryDelayMs}ms`, ); retryActivityEmitted = true; this.emitExecutionEvent({ type: 'activity.observed', observation: { schemaVersion: 1, activityId, target: { kind: 'provider', provider, operation: toolId, label: toolId, }, state: { kind: 'retrying', reason: failure.isRateLimit ? 'rate_limit' : 'provider_error', retryAt: Date.now() + failure.retryDelayMs, attempt: httpFailureAttempt, }, observedAt: Date.now(), }, }); await options?.parkProviderCall?.(); await this.sleepWithCheckpointHeartbeat(failure.retryDelayMs); this.emitExecutionEvent({ type: 'activity.observed', observation: { schemaVersion: 1, activityId, target: { kind: 'provider', provider, operation: toolId, label: toolId, }, state: { kind: 'active' }, observedAt: Date.now(), }, }); continue; } this.log(failure.error.message); throw failure.error; } if (!responseData) { throw createToolHttpError( toolErrorSchemaVersion, `Tool ${toolId} returned an empty successful response body.`, null, response.status, 'repairable', ); } const parsed = parseToolExecuteResponse(toolId, responseData); setSpanAttributes(span, { 'plays.tool_result_kind': parsed.result == null ? 'null' : Array.isArray(parsed.result) ? 'array' : typeof parsed.result, }); toolCallSucceeded = true; return parsed; } }, ); } finally { if (retryActivityEmitted) { this.emitExecutionEvent({ type: 'activity.observed', observation: { schemaVersion: 1, activityId, target: { kind: 'provider', provider, operation: toolId, label: toolId, }, state: toolCallSucceeded ? { kind: 'completed' } : { kind: 'failed', code: 'PROVIDER_CALL_FAILED' }, observedAt: Date.now(), }, }); } if (!toolSlotTransferred) { toolSlot.release(); } } } /** * Resolve the provider backing a tool (from queue-hint metadata) and feed a * server-observed Retry-After back into the Governor's shared pacer. Tools * without hints use the Governor's default `tool:${toolId}` bucket. */ private async reportToolBackpressure( toolId: string, retryAfterMs: number, ): Promise { const hints: readonly PlayQueueHint[] = this.#options.getToolQueueHints ? await this.#options.getToolQueueHints(toolId) : []; const provider = hints[0]?.provider?.trim() || `tool:${toolId}`; await this.resourceGovernor.reportProviderBackpressure({ provider, toolId, retryAfterMs, }); } private async sleepWithCheckpointHeartbeat(ms: number): Promise { const waitMs = Math.max(1, Math.ceil(ms)); const startedAt = Date.now(); while (Date.now() - startedAt < waitMs) { this.#options.onBatchComplete?.(this.checkpoint); const remainingMs = waitMs - (Date.now() - startedAt); await new Promise((resolve) => setTimeout( resolve, Math.min(TOOL_RETRY_HEARTBEAT_INTERVAL_MS, Math.max(1, remainingMs)), ), ); } this.#options.onBatchComplete?.(this.checkpoint); } } export function createPlayContext(options: ContextOptions): PlayContextImpl { return new PlayContextImpl(options); }