import { isAppRuntimeApiCapacityError } from './app-runtime-api'; /** * Runtime persistence-failure circuit breaker. * * When a runtime persistence write fails mid-map (a work-receipt claim/complete * or an output-sheet flush against the tenant Postgres), continuing to dispatch * NEW provider calls only widens the "billed but not durable" gap: every extra * call bills server-side yet has nowhere to land. The latch trips on the first * such failure and is checked by the worker dispatch loops before they dispatch * any further provider call, converting "all billed, half persisted" into * "billed ≈ persisted + small in-flight window". * * The latch is a plain mutable object shared across all row workers of a map * (single isolate, cooperative scheduling — no locking needed). */ export type RuntimePersistenceLatch = { /** True once the first runtime persistence failure has been observed. */ tripped: boolean; /** Formatted first-failure message (first trip wins). */ cause: string | null; /** Provider calls NOT dispatched because the latch was already tripped. */ preventedCallCount: number; /** * True once a full completeReceipt retry ladder has been exhausted in this * map. Bookkeeping only: the shorten decision now keys on `tripped` (see * `shouldShortenCompleteReceiptLadder`) so fresh entrants collapse as soon as * a persistence failure is confirmed rather than only after the first ladder * fully drains. */ latchRetryBudgetExhausted: boolean; }; export function createRuntimePersistenceLatch(): RuntimePersistenceLatch { return { tripped: false, cause: null, preventedCallCount: 0, latchRetryBudgetExhausted: false, }; } /** * Bounded provider-dispatch wave size. * * A single map chunk can hold hundreds of rows. The persistence latch only * trips AFTER a receipt completes (i.e. after the provider call already billed), * so if every row in a big chunk dispatches its provider call in one synchronous * wave, the latch cannot gate anything within that chunk — all calls bill before * the first failure is even observed. That is the 250/250 incident (run * play/x-lima-shard-03/run/20260703t231850-473-44f3d408: a customer billed * 250/250 with preventedCallCount=0, because enrich batches compile to one * ~245-row chunk with effectively unbounded row concurrency). * * Dispatching provider calls in waves of this size, with a single latch read * between waves, bounds billed exposure for ANY chunk shape to ~one wave plus * the calls already in flight when the latch trips. Constant on purpose (house * preference: searchable, no env var). */ export const PROVIDER_DISPATCH_WAVE_SIZE = 25; /** * Partition dispatch units into bounded waves. Each unit contributes `sizeOf` * toward the wave budget: 1 for a single provider call, the batch member count * for a batched group. A batch is atomic and is never split across waves — an * over-budget batch forms its own wave and still counts its full member size * toward the budget. Pure + synchronous so the wave arithmetic is unit-testable * apart from the async dispatch loops that consume it. */ export function partitionDispatchWaves( units: readonly T[], sizeOf: (unit: T) => number, waveSize: number = PROVIDER_DISPATCH_WAVE_SIZE, ): T[][] { const budget = Math.max(1, Math.floor(waveSize)); const waves: T[][] = []; let current: T[] = []; let weight = 0; for (const unit of units) { if (current.length > 0 && weight >= budget) { waves.push(current); current = []; weight = 0; } current.push(unit); weight += Math.max(1, Math.floor(sizeOf(unit))); } if (current.length > 0) waves.push(current); return waves; } function formatLatchCause(error: unknown): string { const raw = error instanceof Error ? error.message : typeof error === 'string' ? error : (() => { try { return JSON.stringify(error); } catch { return String(error); } })(); const message = (raw ?? '').replace(/\s+/g, ' ').trim(); if (!message) return 'Runtime persistence failed.'; return message.length > 1_000 ? `${message.slice(0, 1_000)}…` : message; } /** * Trip the latch on the first persistence failure. Idempotent: later trips are * no-ops so `cause` keeps the first, root failure. */ export function tripRuntimePersistenceLatch( latch: RuntimePersistenceLatch, error: unknown, ): void { if (isAppRuntimeApiCapacityError(error)) return; if (latch.tripped) return; latch.tripped = true; latch.cause = formatLatchCause(error); } /** * Thrown in place of dispatching a provider call once the latch is tripped. The * message deliberately contains the substring `persistence failure` so that a * `failed` receipt carrying this error is reclaimable on resume (see * `canReclaimFailedWorkerToolReceipt`). */ export class RuntimePersistenceCircuitOpenError extends Error { constructor(latch: RuntimePersistenceLatch) { super( `Runtime persistence failure circuit breaker: skipped dispatching this provider call ` + `because a runtime persistence failure already occurred in this map. ` + `First persistence error: ${latch.cause}`, ); this.name = 'RuntimePersistenceCircuitOpenError'; } } /** * Decide whether a completeReceipt retry ladder entrant should collapse to a * single attempt instead of running the full backoff ladder. * * Keyed on `latch.tripped` (NOT `latchRetryBudgetExhausted`): the latch only * trips once a persistence failure is CONFIRMED — either a full completeReceipt * ladder already gave up, or an output-sheet flush failed. A transient blip that * a ladder rides out never trips the latch, so the first ladder always keeps its * full budget and blip tolerance is preserved. Once the latch is tripped the * endpoint is known-dead, so every FRESH entrant collapses to one attempt rather * than piling another full ladder of nested connect attempts onto a saturated * database. A ladder that already committed to retrying (`committedToFullLadder`) * is protected: it keeps its full budget so an in-progress leader is never * truncated mid-flight by a peer that tripped the latch a moment later. */ export function shouldShortenCompleteReceiptLadder( latch: RuntimePersistenceLatch, committedToFullLadder: boolean, ): boolean { return latch.tripped && !committedToFullLadder; } export function isRuntimePersistenceCircuitOpenError(error: unknown): boolean { if (!error || typeof error !== 'object') return false; if (error instanceof RuntimePersistenceCircuitOpenError) return true; return ( error instanceof Error && error.name === 'RuntimePersistenceCircuitOpenError' ); }