/** * Salvage of orphaned billed tool results. * * When a provider call has billed server-side but its durable receipt cannot be * marked completed (tenant Postgres unwritable) even after the retry ladder, * the serialized tool output would otherwise live only in isolate memory and be * lost when the run fails. This module collects those outputs and builds a * size-capped salvage record that rides on the run.failed ledger event's * `result` field — a different durable system (Convex run ledger) than the dead * tenant Postgres. * * The salvage is a LOUD marker on a FAILED run, never a fallback that lets the * run succeed. Entries carry exactly the bytes a receipt would have stored * (customer-owned tool output); no provider-spend / cost metadata is added. */ /** Total salvage payload cap. Well under Convex's ~1MB document limit once the * rest of the failure result + run snapshot are accounted for. */ export const SALVAGE_MAX_TOTAL_BYTES = 262_144; /** Per-entry output cap: an oversized single output is dropped (metadata kept). */ export const SALVAGE_MAX_ENTRY_BYTES = 32_768; export type ReceiptSalvageEntry = { receiptKey: string; toolId: string; cacheKey?: string; /** Serialized tool output (what completeReceipt would have persisted). * Omitted when the output was dropped for size. */ output?: unknown; /** UTF-8 byte length of the serialized output (0 when there was none). */ outputBytes: number; /** True when this entry's output was dropped for size. */ truncated: boolean; /** completeReceipt attempts made before giving up. */ completeAttempts: number; /** epoch ms of the first completion failure for this entry. */ firstErrorAt: number; }; export type ReceiptSalvage = { entries: ReceiptSalvageEntry[]; /** Total orphaned results collected, including any dropped entirely. */ totalEntries: number; /** Entries dropped entirely (not even metadata retained) for total-size cap. */ droppedEntries: number; /** True when any output was dropped or any entry was dropped entirely. */ truncated: boolean; /** Loud human-readable marker (see note strings below). */ note: string; }; export type ReceiptSalvageInput = { receiptKey: string; toolId: string; cacheKey?: string; /** Serialized durable output (already JSON-safe, as receipts store it). */ output: unknown; completeAttempts: number; firstErrorAt: number; }; function jsonByteLength(value: unknown): number { let json: string | undefined; try { json = JSON.stringify(value); } catch { return 0; } if (!json) return 0; return new TextEncoder().encode(json).length; } /** * Cap-enforcing, pure builder. Keeps metadata-only entries until even metadata * would exceed the total cap, then drops remaining entries entirely and counts * them in `droppedEntries`. */ export function buildReceiptSalvage( inputs: readonly ReceiptSalvageInput[], ): ReceiptSalvage | null { if (inputs.length === 0) return null; const totalEntries = inputs.length; const entries: ReceiptSalvageEntry[] = []; // Reserve the enclosing `[` and `]`. Every appended entry also reserves one // byte for the joining comma, so `usedBytes` stays a conservative upper bound // on the serialized `entries` array length. let usedBytes = 2; let droppedEntries = 0; let truncatedOutputs = 0; for (const input of inputs) { const outputBytes = jsonByteLength(input.output); const metaOnly: ReceiptSalvageEntry = { receiptKey: input.receiptKey, toolId: input.toolId, ...(input.cacheKey ? { cacheKey: input.cacheKey } : {}), outputBytes, truncated: outputBytes > 0, completeAttempts: input.completeAttempts, firstErrorAt: input.firstErrorAt, }; const metaBytes = jsonByteLength(metaOnly) + 1; // Even the metadata-only footprint would blow the total cap: drop entirely. if (usedBytes + metaBytes > SALVAGE_MAX_TOTAL_BYTES) { droppedEntries += 1; continue; } const withOutput: ReceiptSalvageEntry = { ...metaOnly, truncated: false, output: input.output, }; const fullBytes = jsonByteLength(withOutput) + 1; const canIncludeOutput = outputBytes > 0 && outputBytes <= SALVAGE_MAX_ENTRY_BYTES && usedBytes + fullBytes <= SALVAGE_MAX_TOTAL_BYTES; if (canIncludeOutput) { entries.push(withOutput); usedBytes += fullBytes; } else { entries.push(metaOnly); if (outputBytes > 0) truncatedOutputs += 1; usedBytes += metaBytes; } } const truncated = droppedEntries > 0 || truncatedOutputs > 0; const note = truncated ? `SALVAGE TRUNCATED: ${truncatedOutputs + droppedEntries} of ${totalEntries} billed-but-unpersisted tool outputs exceeded the salvage size cap and were dropped. All listed receiptKeys were billed; re-running will re-execute unpersisted calls.` : `${totalEntries} billed tool result(s) could not be durably persisted; serialized outputs are attached for manual export.`; return { entries, totalEntries, droppedEntries, truncated, note }; } /** Loud one-line suffix appended to the run.failed error message. */ export function receiptSalvageFailureSuffix(salvage: ReceiptSalvage): string { return `${salvage.totalEntries} provider result(s) were billed but could not be persisted; salvage attached to the run record.`; } /** * Run-scoped collection buffer. Thin wrapper over an array so a single * reference can be threaded to every scheduler in the run; `build()` runs the * pure cap-enforcing builder at terminal time. */ export type ReceiptSalvageBuffer = { push(input: ReceiptSalvageInput): void; size(): number; build(): ReceiptSalvage | null; }; export function createReceiptSalvageBuffer(): ReceiptSalvageBuffer { const inputs: ReceiptSalvageInput[] = []; return { push: (input) => { inputs.push(input); }, size: () => inputs.length, build: () => buildReceiptSalvage(inputs), }; }