import { getAddress } from 'viem' import * as Db from '../../db/Db.js' import * as RoutesTransferTransactions from '../../db/tables/routesTransferTransactions.js' import * as RoutesTransfers from '../../db/tables/routesTransfers.js' import type * as Metrics from '../../Metrics.js' import * as MetricSink from '../MetricSink.js' import * as Value from '../Value.js' import type * as Webhooks from '../Webhooks.js' import * as Catalog from './Catalog.js' import * as Observability from './Observability.js' import * as Provider from './Provider.js' import * as Transfer from './Transfer.js' import * as TransferSubsidy from './TransferSubsidy.js' const followUpDelaySeconds = 2 const maximumFollowUpAttempts = 900 const sweepLimit = 25 /** Queue message that advances one registered route transfer. */ export type Message = { /** Zero-based destination reconciliation attempt. */ attempt?: number | undefined /** Route transfer id (`rtr_…`). */ transferId: string /** Route transfer reconciliation discriminator. */ type: 'routes:transfer:reconcile' } /** Creates a tracker that verifies and completes registered route transfers. */ export function createTracker(options: createTracker.Options): createTracker.Tracker { const providers = options.providers.filter(Provider.canReconcileTransfer) const providersById = new Map(providers.map((provider) => [provider.id, provider])) const recordProviderOperation = MetricSink.routesProviderOperations(options.metrics) const sweepCursors = new Map() const record = (outcome: string, providerId: string, count = 1) => { options.metrics?.count('routes_transfer_reconciliation_count', count, { outcome, provider: providerId, }) } const tracker: createTracker.Tracker = { async reconcile(message) { const db = Db.get(options.db) const transfer = await RoutesTransfers.get(db, message.transferId) if (!transfer || transfer.status === 'completed') { record(transfer ? 'already_completed' : 'not_found', transfer?.providerId ?? 'unresolved') return { type: 'ignored' } } const provider = providersById.get(transfer.providerId) const recordOutcome = (outcome: string) => record(outcome, transfer.providerId) const recordSubsidy = (outcome: string) => options.metrics?.count('routes_transfer_subsidy_count', 1, { outcome, provider: transfer.providerId, }) if (transfer.status !== 'processing' || !provider) { recordOutcome('invalid_state') return { type: 'ignored' } } const sourceTransactionRef = transfer.snapshot.sourceTransactionHashes?.[0] const source = sourceTransactionRef ? await RoutesTransferTransactions.getSource(db, { chainId: transfer.snapshot.sourceChain.id, transactionRef: sourceTransactionRef, transferId: transfer.id, }) : undefined const catalog = await Catalog.read(db) const destinationChain = catalog.chainsByKey.get(transfer.snapshot.destinationChain.id) const sourceChain = catalog.chainsByKey.get(transfer.snapshot.sourceChain.id) const destinationAmount = transfer.snapshot.destinationAmount if ( !source || source.transferId !== transfer.id || !destinationChain || !destinationAmount || !sourceChain ) throw new EvidenceError('Registered route transfer evidence is incomplete.') const signal = new AbortController().signal const delivery = await Provider.observe( provider, () => provider.verifyDestinationDelivery( { destinationChain, providerState: transfer.providerState ?? undefined, sourceChain, transfer: transfer.snapshot, }, signal, ), { onAttempt: recordProviderOperation, operation: 'verifyDestinationDelivery' }, ) if (delivery.type === 'invalid') throw new EvidenceError('Tempo delivery conflicts with registered transfer terms.') if (delivery.type === 'pending') { const verified = await Provider.observe( provider, () => provider.verifySourceTransaction( { destinationChain, providerState: transfer.providerState ?? undefined, sourceChain, transactionHash: source.transactionRef, transfer: transfer.snapshot, }, signal, ), { onAttempt: recordProviderOperation, operation: 'verifySourceTransaction' }, ) if (verified.type === 'reorged') { const reopened = await Transfer.reopenSourceTransaction(db, { chainId: transfer.snapshot.sourceChain.id, id: transfer.id, providerState: verified.providerState, transactionHash: source.transactionRef, }) if (reopened.type !== 'reopened') { recordOutcome('invalid_state') return { type: 'ignored' } } Observability.recordTransfer(options.metrics, { previous: transfer, record: reopened.record, }) if (options.dispatchTransferUpdates && reopened.references.length > 0) await options.dispatchTransferUpdates(reopened.references) recordOutcome('reopened') return { type: 'reopened' } } if (verified.type === 'pending') return followUp(message, options, recordOutcome) if (verified.type === 'invalid' || verified.type === 'related') throw new EvidenceError('Source transaction conflicts with registered provider evidence.') return followUp(message, options, recordOutcome) } const providerDelivery = await Transfer.recordProviderDelivery(db, { id: transfer.id }) if (providerDelivery.type === 'invalid' || providerDelivery.type === 'not_found') { recordOutcome('invalid_state') return { type: 'ignored' } } if (providerDelivery.type === 'recorded') { Observability.recordTransfer(options.metrics, { previous: transfer, record: providerDelivery.record, }) } let current = providerDelivery.record const transactionHashes: [string, ...string[]] = [delivery.transactionHash] let deliveredAmount = destinationAmount if (transfer.snapshot.subsidize) { const required = transfer.snapshot.destinationAmountRequired if (!required) throw new EvidenceError('Normalized destination amount is missing.') const deficit = BigInt(required.baseUnits) - BigInt(destinationAmount.baseUnits) // A fully covered quote may have no subsidy signer commitment. if (deficit > 0n) { let transaction = TransferSubsidy.read(current.providerState?.['transferSubsidy']) const settler = transaction ? options.subsidies?.[0] : options.subsidies?.find( ({ account }) => account.toLowerCase() === current.subsidyAccount?.toLowerCase(), ) if (!settler) { recordSubsidy('unavailable') return actionRequired(current, options, { code: 'subsidy_failed', message: 'Destination subsidy settlement is unavailable.', }) } if (!transaction) { const commitment = current.subsidyCommitmentAmount if (!commitment || deficit > BigInt(commitment.baseUnits)) { recordSubsidy('routes_subsidy_limit_exceeded') return actionRequired(current, options, { code: 'routes_subsidy_limit_exceeded', message: 'The required destination subsidy exceeds its approved commitment.', }) } const pendingNonce = await settler.pendingNonce({ chain: destinationChain }) const subsidyAmount = Value.tokenAmount({ baseUnits: deficit, currency: transfer.snapshot.destinationToken.currency, decimals: transfer.snapshot.destinationToken.decimals, }) const reserved = await Transfer.reserveSubsidy(db, { account: settler.account, chainId: destinationChain.id, id: current.id, orgId: current.orgId, pendingNonce, prepare: ({ nonce, reservations }) => settler.prepare({ amount: deficit, chain: destinationChain, nonce, recipient: getAddress(transfer.snapshot.recipient), reservations, token: getAddress(transfer.snapshot.destinationToken.address), }), subsidyAmount, tokenAddress: transfer.snapshot.destinationToken.address, }).catch((cause) => { if (cause instanceof Transfer.SubsidyEvidenceError) { recordSubsidy('evidence_conflict') throw cause } if (cause instanceof TransferSubsidy.BalanceUnavailableError) return undefined if (cause instanceof Transfer.SubsidySignerBlockedError) return 'blocked' as const if (TransferSubsidy.isPermanentPreparationError(cause)) return 'rejected' as const throw cause }) if (reserved === 'blocked') { recordSubsidy('signer_blocked') return actionRequired(current, options, { code: 'subsidy_failed', message: 'Destination subsidy settlement requires operator recovery.', }) } if (reserved === 'rejected') { recordSubsidy('prepare_rejected') return actionRequired(current, options, { code: 'subsidy_failed', message: 'The destination subsidy transaction could not be prepared.', }) } if (!reserved) { recordSubsidy('balance_unavailable') return actionRequired(current, options, { code: 'subsidy_balance_unavailable', message: 'Tempo lacks enough destination token inventory or native gas.', }) } if (reserved.type === 'invalid' || reserved.type === 'not_found') { recordOutcome('invalid_state') return { type: 'ignored' } } transaction = reserved.transaction current = reserved.record if ( (reserved.type === 'reserved' || reserved.type === 'shared') && options.dispatchTransferUpdates && reserved.references.length > 0 ) await options.dispatchTransferUpdates(reserved.references) if (reserved.type === 'reserved') recordSubsidy('prepared') if (reserved.type === 'shared') recordSubsidy('shared') } const status = await settler.status({ chain: destinationChain, recipient: getAddress(transfer.snapshot.recipient), token: getAddress(transfer.snapshot.destinationToken.address), transaction, }) if (status.type === 'reverted') { recordSubsidy('reverted') return actionRequired( current, options, { code: 'subsidy_failed', message: 'The destination subsidy transaction reverted.', }, { subsidyAmount: Value.tokenAmount({ baseUnits: 0n, currency: transfer.snapshot.destinationToken.currency, decimals: transfer.snapshot.destinationToken.decimals, }), }, ) } if (status.type === 'pending') { if (status.broadcast) { const rejected = await settler .broadcast({ chain: destinationChain, transaction }) .then(() => false) .catch((cause) => { if (alreadyKnown(cause)) return false if ( cause instanceof TransferSubsidy.TransactionHashMismatchError || TransferSubsidy.isPermanentBroadcastError(cause) ) return true recordSubsidy('broadcast_failed') throw cause }) if (rejected) { recordSubsidy('broadcast_rejected') const blocked = await Transfer.blockSubsidy(db, { id: current.id, statusReason: { code: 'subsidy_failed', message: 'The destination subsidy transaction was rejected.', }, }) if (blocked.type === 'applied') { Observability.recordTransfer(options.metrics, { previous: current, record: blocked.record, }) if (options.dispatchTransferUpdates && blocked.references.length > 0) await options.dispatchTransferUpdates(blocked.references) } return { type: 'ignored' } } recordSubsidy('broadcast') } return followUp(message, options, recordOutcome) } deliveredAmount = Value.tokenAmount({ baseUnits: BigInt(destinationAmount.baseUnits) + BigInt(transaction.amount), currency: destinationAmount.currency, decimals: destinationAmount.decimals, }) transactionHashes.push(transaction.hash) recordSubsidy('completed') } } const completed = await Transfer.complete(db, { chainId: current.snapshot.destinationChain.id, destinationAmount: deliveredAmount, id: current.id, transactionHashes, }) if (completed.type === 'invalid') throw new EvidenceError('Route transfer cannot accept destination evidence.') if (completed.type === 'not_found') { recordOutcome('not_found') return { type: 'ignored' } } if (completed.type === 'completed') Observability.recordTransfer(options.metrics, { previous: current, record: completed.record, }) if ( completed.type === 'completed' && options.dispatchTransferUpdates && completed.references.length > 0 ) await options.dispatchTransferUpdates(completed.references) recordOutcome(completed.type) return { type: 'completed' } }, async tick() { if (!options.dispatch) throw new DispatchError() const db = Db.get(options.db) let checked = 0 const abandoned = await RoutesTransfers.listExpiredSubsidyCommitments(db, { expiredBefore: new Date( Date.now() - RoutesTransfers.subsidyCommitmentRecoveryTtlMs, ).toISOString(), limit: sweepLimit, }) for (const transfer of abandoned) { const result = await Transfer.transition(db, { expectedVersion: transfer.version, id: transfer.id, status: 'expired', }) if (result.type !== 'applied') continue Observability.recordTransfer(options.metrics, { previous: transfer, record: result.record }) record('expired', transfer.providerId) if (options.dispatchTransferUpdates && result.references.length > 0) await options.dispatchTransferUpdates(result.references) } for (const provider of providers) { let cursor = sweepCursors.get(provider.id) let records = await RoutesTransfers.listProcessing(db, { cursor, limit: sweepLimit, providerId: provider.id, }) if (records.length === 0 && cursor !== undefined) { cursor = undefined sweepCursors.delete(provider.id) records = await RoutesTransfers.listProcessing(db, { limit: sweepLimit, providerId: provider.id, }) } for (const transfer of records) { await options.dispatch( { transferId: transfer.id, type: 'routes:transfer:reconcile', }, { delaySeconds: 0 }, ) cursor = transfer.id sweepCursors.set(provider.id, cursor) checked++ } record('swept', provider.id, records.length) } return { checked } }, } const metrics = options.metrics if (!metrics) return tracker return { ...tracker, async reconcile(message) { const startedAt = performance.now() try { const result = await tracker.reconcile(message) metrics.histogram( 'routes_transfer_reconciliation_duration_ms', performance.now() - startedAt, { outcome: result.type }, ) return result } catch (cause) { metrics.histogram( 'routes_transfer_reconciliation_duration_ms', performance.now() - startedAt, { outcome: 'failed' }, ) throw cause } finally { metrics.flush() } }, } } export declare namespace createTracker { /** Tracker dependencies and queue dispatch. */ type Options = { /** Authoritative database or per-invocation database factory. */ db: Db.Source /** Enqueues one delayed reconciliation follow-up. */ dispatch?: ((message: Message, options: { delaySeconds: number }) => Promise) | undefined /** Enqueues webhook updates staged with transfer mutations. */ dispatchTransferUpdates?: | ((references: readonly Webhooks.QueueReference[]) => Promise) | undefined /** Optional bounded operational metrics. */ metrics?: Metrics.Metrics | undefined /** Route providers available for transfer reconciliation. */ providers: readonly Provider.Provider[] /** Destination-chain subsidy settlers, active first with retained rotation keys after it. */ subsidies?: readonly TransferSubsidy.Settler[] | undefined } /** Reconciles one registered route transfer. */ type Tracker = { /** Verifies or defers one destination delivery. */ reconcile(message: Message): Promise /** Expires abandoned commitments and enqueues processing transfers. */ tick(): Promise<{ checked: number }> } /** One reconciliation disposition. */ type Result = | { type: 'completed' } | { type: 'ignored' } | { type: 'pending' } | { type: 'reopened' } } async function actionRequired( transfer: RoutesTransfers.Record, options: createTracker.Options, statusReason: Transfer.StatusReason, state: Pick = {}, ): Promise { const result = await Transfer.transition(Db.get(options.db), { expectedVersion: transfer.version, id: transfer.id, ...state, status: 'action-required', statusReason, }) if (result.type === 'applied') { Observability.recordTransfer(options.metrics, { previous: transfer, record: result.record }) if (options.dispatchTransferUpdates && result.references.length > 0) await options.dispatchTransferUpdates(result.references) } return { type: 'ignored' } } async function followUp( message: Message, options: createTracker.Options, record: (outcome: string) => void, ): Promise { const attempt = (message.attempt ?? 0) + 1 if (attempt > maximumFollowUpAttempts) { record('exhausted') throw new ExhaustedError() } if (!options.dispatch) throw new DispatchError() await options.dispatch( { attempt, transferId: message.transferId, type: 'routes:transfer:reconcile' }, { delaySeconds: followUpDelaySeconds }, ) record('pending') return { type: 'pending' } } class DispatchError extends Error { override name = 'Routes.TransferReconciliation.DispatchError' } class EvidenceError extends Error { override name = 'Routes.TransferReconciliation.EvidenceError' } function alreadyKnown(cause: unknown): boolean { if (!(cause instanceof Error)) return false const message = `${cause.message} ${'details' in cause ? String(cause.details) : ''}`.toLowerCase() return message.includes('already known') || message.includes('known transaction') } class ExhaustedError extends Error { override name = 'Routes.TransferReconciliation.ExhaustedError' }