import crypto from "crypto"; import fs from "fs"; import path from "path"; import { Interface, JsonRpcProvider } from "ethers"; import { loadReleaseExecutionPlan, ReleaseExecutionPlan, ReleaseExecutionTransaction, } from "./executionPlan"; import { loadExternalExecutionPackage, type ExternalExecutionPackage, } from "./external"; import { loadOperatorExecutionResult } from "./operatorExecution"; import { markDeploymentRecordReconciled } from "./deploymentRecords"; import { loadConfig, resolveFinalityConfirmations, resolveTarget } from "./config"; export type ExternalFinalityPolicy = | { mode: "rpc_finalized" } | { mode: "confirmation_depth"; confirmations: number }; export interface ExternalReconciliationMatch { transactionId: string; transactionIndex: number; route: "operator" | "external"; executor: string; submittedBy: string; executionKind: "eoa" | "safe_call" | "safe_multisend"; transactionHash: string; blockNumber: number; blockHash: string; blockTransactionIndex: number; innerCallIndex?: number; receiptStatus: 0 | 1; finalized: boolean; } export interface ExternalReconciliationReport { version: 1; status: "not_required" | "waiting_operator" | "waiting_finality" | "waiting_external" | "reconciled" | "failed"; taskId: string; target: string; executionId: string; executionPlanHash: string; packagePath?: string; packageHash?: string; segments?: Array<{ segmentId: string; transactionIds: string[]; handoffBlock: number; nextBlock: number; }>; finalizedBlock: number; finalityPolicy: ExternalFinalityPolicy; matchedTransactionIds: string[]; matches: ExternalReconciliationMatch[]; failure?: { code: string; message: string; transactionId?: string }; reportPath: string; createdAt: string; updatedAt: string; completedAt?: string; } interface RpcTransaction { hash: string; from: string; to: string | null; value: unknown; data?: string; input?: string; blockNumber?: unknown; blockHash?: string; index?: unknown; transactionIndex?: unknown; } interface RpcReceipt { status: unknown; blockNumber: unknown; blockHash: string; index?: unknown; transactionIndex?: unknown; from?: string; to?: string | null; logs?: Array<{ address?: string; topics?: string[]; data?: string; }>; } interface ReconciliationProvider { getBlockNumber(): Promise; getCode(address: string): Promise; getTransaction(hash: string): Promise; getTransactionReceipt(hash: string): Promise; send(method: string, params: unknown[]): Promise; call?(transaction: { to: string; data: string }): Promise; } const SAFE_INTERFACE = new Interface([ "function execTransaction(address to,uint256 value,bytes data,uint8 operation,uint256 safeTxGas,uint256 baseGas,uint256 gasPrice,address gasToken,address payable refundReceiver,bytes signatures) returns (bool success)", "function VERSION() view returns (string)", "event ExecutionSuccess(bytes32 txHash,uint256 payment)", "event ExecutionFailure(bytes32 txHash,uint256 payment)", ]); const MULTISEND_INTERFACE = new Interface([ "function multiSend(bytes transactions)", ]); const sha256 = (value: Buffer | string): string => ( crypto.createHash("sha256").update(value).digest("hex") ); const numberValue = (value: unknown, label: string): number => { let parsed: bigint; try { parsed = typeof value === "bigint" ? value : BigInt(value as string | number); } catch { throw new Error(`${label} is not an integer`); } if (parsed < 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) { throw new Error(`${label} is outside the safe integer range`); } return Number(parsed); }; const address = (value: unknown, label: string): string => { if (typeof value !== "string" || !/^0x[0-9a-fA-F]{40}$/.test(value)) { throw new Error(`${label} is not an EVM address`); } return value.toLowerCase(); }; const calldata = (transaction: RpcTransaction): string => { const value = transaction.data ?? transaction.input; if (typeof value !== "string" || !/^0x(?:[0-9a-fA-F]{2})*$/.test(value)) { throw new Error(`Transaction ${transaction.hash} has invalid calldata`); } return value.toLowerCase(); }; const receiptStatus = (receipt: RpcReceipt): 0 | 1 => { const status = numberValue(receipt.status, "receipt status"); if (status !== 0 && status !== 1) throw new Error("receipt status must be 0 or 1"); return status; }; const relative = (root: string, filePath: string): string => { const value = path.relative(path.resolve(root), path.resolve(filePath)); if (value.startsWith("..") || path.isAbsolute(value)) { throw new Error("Reconciliation evidence path escapes the project root"); } return value.split(path.sep).join("/"); }; const readJson = (filePath: string, label: string): unknown => { try { return JSON.parse(fs.readFileSync(filePath, "utf8")); } catch (error) { throw new Error(`${label} is invalid: ${error instanceof Error ? error.message : String(error)}`); } }; const writeJsonAtomic = (filePath: string, value: unknown): void => { fs.mkdirSync(path.dirname(filePath), { recursive: true }); const temporary = `${filePath}.${process.pid}.${Date.now()}.tmp`; try { fs.writeFileSync(temporary, JSON.stringify(value, null, 2) + "\n"); fs.renameSync(temporary, filePath); } catch (error) { if (fs.existsSync(temporary)) fs.unlinkSync(temporary); throw error; } }; const loadExternalPackage = (input: { root: string; taskDir: string; plan: ReleaseExecutionPlan; }): { package: ExternalExecutionPackage; filePath: string; hash: string } | undefined => { const externalTransactions = input.plan.transactions.filter((transaction) => ( transaction.route === "external" )); const filePath = path.join( input.taskDir, "calldata", input.plan.target, input.plan.executionId, "external-execution.json", ); if (externalTransactions.length === 0) { if (fs.existsSync(filePath)) { throw new Error("All-operator execution must not contain an external package"); } return undefined; } const loaded = loadExternalExecutionPackage({ taskDir: input.taskDir, plan: input.plan }); if (!loaded) return undefined; return { package: loaded.package, filePath: loaded.filePath, hash: sha256(fs.readFileSync(loaded.filePath)), }; }; const transactionMatches = ( transaction: RpcTransaction, expected: ReleaseExecutionTransaction, from: string, ): boolean => ( address(transaction.from, "transaction sender") === address(from, "expected sender") && transaction.to !== null && address(transaction.to, "transaction target") === expected.to && numberValue(transaction.value, "transaction value").toString() === expected.value && calldata(transaction) === expected.data && expected.operation === 0 ); const parseMultisend = (encoded: string): Array<{ operation: 0 | 1; to: string; value: string; data: string; }> => { const payload = encoded.startsWith("0x") ? encoded.slice(2) : encoded; const calls: Array<{ operation: 0 | 1; to: string; value: string; data: string }> = []; let offset = 0; while (offset < payload.length) { if (payload.length - offset < 2 + 40 + 64 + 64) { throw new Error("Safe MultiSend payload is truncated"); } const operation = Number.parseInt(payload.slice(offset, offset + 2), 16); offset += 2; if (operation !== 0 && operation !== 1) throw new Error("Safe MultiSend operation is invalid"); const to = `0x${payload.slice(offset, offset + 40)}`.toLowerCase(); offset += 40; const value = BigInt(`0x${payload.slice(offset, offset + 64)}`).toString(10); offset += 64; const length = numberValue(`0x${payload.slice(offset, offset + 64)}`, "MultiSend data length"); offset += 64; const end = offset + length * 2; if (end > payload.length) throw new Error("Safe MultiSend call data is truncated"); calls.push({ operation, to, value, data: `0x${payload.slice(offset, end)}`.toLowerCase() }); offset = end; } return calls; }; const safeCalls = async ( provider: ReconciliationProvider, transaction: RpcTransaction, safeAddress: string, ): Promise> => { if (address(transaction.to, "Safe transaction target") !== safeAddress) return []; if (!provider.call) throw new Error(`Unknown contract-wallet executor ${safeAddress}`); try { const versionResult = await provider.call({ to: safeAddress, data: SAFE_INTERFACE.encodeFunctionData("VERSION", []), }); SAFE_INTERFACE.decodeFunctionResult("VERSION", versionResult); } catch { throw new Error(`Unknown contract-wallet executor ${safeAddress}`); } let decoded; try { decoded = SAFE_INTERFACE.decodeFunctionData("execTransaction", calldata(transaction)); } catch { return []; } const outer = { to: address(decoded[0], "Safe inner target"), value: BigInt(decoded[1]).toString(10), data: String(decoded[2]).toLowerCase(), operation: Number(decoded[3]) as 0 | 1, }; try { const multisend = MULTISEND_INTERFACE.decodeFunctionData("multiSend", outer.data); return parseMultisend(String(multisend[0])).map((call, innerCallIndex) => ({ ...call, kind: "safe_multisend" as const, innerCallIndex, })); } catch { return [{ ...outer, kind: "safe_call" }]; } }; const assertSafeExecutionSucceeded = (receipt: RpcReceipt, safeAddress: string): void => { let success = false; let failure = false; for (const log of receipt.logs || []) { if ( typeof log.address !== "string" || log.address.toLowerCase() !== safeAddress.toLowerCase() || !Array.isArray(log.topics) || typeof log.data !== "string" ) { continue; } try { const parsed = SAFE_INTERFACE.parseLog({ topics: log.topics, data: log.data }); if (parsed?.name === "ExecutionFailure") failure = true; if (parsed?.name === "ExecutionSuccess") success = true; } catch { // Ignore unrelated logs emitted by the Safe transaction. } } if (failure) throw new Error(`Safe ${safeAddress} emitted ExecutionFailure`); if (!success) throw new Error(`Safe ${safeAddress} receipt is missing ExecutionSuccess`); }; const finalizedHeight = async ( provider: ReconciliationProvider, confirmations: number, ): Promise<{ finalizedBlock: number; policy: ExternalFinalityPolicy }> => { try { const block = await provider.send("eth_getBlockByNumber", ["finalized", false]) as { number?: unknown; } | null; if (block?.number !== undefined) { return { finalizedBlock: numberValue(block.number, "finalized block number"), policy: { mode: "rpc_finalized" }, }; } } catch { // Some EVM RPC providers do not implement the finalized block tag. } const latest = await provider.getBlockNumber(); if (!Number.isSafeInteger(latest) || latest < 0) { throw new Error("RPC latest block number is invalid"); } return { finalizedBlock: Math.max(0, latest - confirmations), policy: { mode: "confirmation_depth", confirmations }, }; }; const receiptMatch = ( expected: ReleaseExecutionTransaction, transaction: RpcTransaction, receipt: RpcReceipt, finalizedBlock: number, details: { executionKind: "eoa" | "safe_call" | "safe_multisend"; submittedBy: string; innerCallIndex?: number; }, ): ExternalReconciliationMatch => { const blockNumber = numberValue(receipt.blockNumber, "receipt blockNumber"); const blockTransactionIndex = numberValue( receipt.index ?? receipt.transactionIndex ?? transaction.index ?? transaction.transactionIndex ?? 0, "receipt transaction index", ); return { transactionId: expected.transactionId, transactionIndex: expected.transactionIndex, route: expected.route, executor: expected.executor, submittedBy: address(details.submittedBy, "submittedBy"), executionKind: details.executionKind, transactionHash: transaction.hash.toLowerCase(), blockNumber, blockHash: receipt.blockHash.toLowerCase(), blockTransactionIndex, innerCallIndex: details.innerCallIndex, receiptStatus: receiptStatus(receipt), finalized: blockNumber <= finalizedBlock, }; }; const validateCanonicalReceipt = async ( provider: ReconciliationProvider, receipt: RpcReceipt, ): Promise => { const blockNumber = numberValue(receipt.blockNumber, "receipt blockNumber"); const block = await provider.send("eth_getBlockByNumber", [`0x${blockNumber.toString(16)}`, false]) as { hash?: string; } | null; if (!block?.hash || block.hash.toLowerCase() !== receipt.blockHash.toLowerCase()) { throw new Error("Transaction receipt is no longer in the canonical chain"); } }; export const reconcileExternalExecution = async (input: { root: string; taskId: string; network: string; executionId: string; provider: JsonRpcProvider; batchSize?: number; }): Promise => { const provider = input.provider as unknown as ReconciliationProvider; const taskDir = path.join(input.root, "scripts", "tasks", input.taskId); const plan = loadReleaseExecutionPlan({ root: input.root, taskDir, target: input.network, executionId: input.executionId, }); if (plan.taskId !== input.taskId) throw new Error("Execution plan taskId mismatch"); const packageEvidence = loadExternalPackage({ root: input.root, taskDir, plan }); const operatorResultPath = path.join( taskDir, "results", plan.target, plan.executionId, "operator-execution.json", ); const operatorResult = fs.existsSync(operatorResultPath) ? loadOperatorExecutionResult({ taskDir, plan }) : undefined; const reportFile = path.join( taskDir, "results", input.network, input.executionId, "external-reconciliation.json", ); const reportPath = relative(input.root, reportFile); const previous = fs.existsSync(reportFile) ? readJson(reportFile, "external reconciliation report") as ExternalReconciliationReport : undefined; if ( previous && ( previous.version !== 1 || previous.taskId !== input.taskId || previous.target !== input.network || previous.executionId !== input.executionId || previous.executionPlanHash !== plan.planHash ) ) { throw new Error("Persisted reconciliation report does not match this execution"); } const target = resolveTarget(loadConfig(input.root), input.network); const confirmations = resolveFinalityConfirmations(target, input.network); const finality = await finalizedHeight(provider, confirmations); const { finalizedBlock } = finality; const matches = new Map(); let failure: ExternalReconciliationReport["failure"]; try { for (const evidence of operatorResult?.receipts ?? []) { const expected = plan.transactions.find((transaction) => ( transaction.transactionId === evidence.transactionId )); if (!expected || expected.route !== "operator") { throw new Error(`Operator receipt ${evidence.transactionId} does not match the plan`); } const transaction = await provider.getTransaction(evidence.transactionHash); const receipt = await provider.getTransactionReceipt(evidence.transactionHash); if (!transaction || !receipt) continue; await validateCanonicalReceipt(provider, receipt); if (!transactionMatches(transaction, expected, expected.executor)) { throw new Error(`Operator transaction ${expected.transactionId} does not match the plan`); } matches.set(expected.transactionId, receiptMatch( expected, transaction, receipt, finalizedBlock, { executionKind: "eoa", submittedBy: transaction.from }, )); } for (const previousMatch of previous?.matches || []) { if (previousMatch.route !== "external") continue; const expected = plan.transactions[previousMatch.transactionIndex]; if (!expected || expected.transactionId !== previousMatch.transactionId) { throw new Error("Persisted external match does not belong to the execution plan"); } const transaction = await provider.getTransaction(previousMatch.transactionHash); const receipt = await provider.getTransactionReceipt(previousMatch.transactionHash); if (!transaction || !receipt) { throw new Error(`Previously matched transaction disappeared: ${previousMatch.transactionHash}`); } await validateCanonicalReceipt(provider, receipt); if (previousMatch.executionKind === "eoa") { if (!transactionMatches(transaction, expected, expected.executor)) { throw new Error(`Previously matched transaction changed: ${expected.transactionId}`); } } else { const decoded = await safeCalls(provider, transaction, expected.executor); const call = decoded[previousMatch.innerCallIndex || 0]; if ( !call || call.to !== expected.to || call.value !== expected.value || call.data !== expected.data || call.operation !== expected.operation ) { throw new Error(`Previously matched Safe transaction changed: ${expected.transactionId}`); } assertSafeExecutionSucceeded(receipt, expected.executor); } matches.set(expected.transactionId, receiptMatch( expected, transaction, receipt, finalizedBlock, { executionKind: previousMatch.executionKind, submittedBy: transaction.from, innerCallIndex: previousMatch.innerCallIndex, }, )); } } catch (error) { failure = { code: "EVIDENCE_INVALID", message: error instanceof Error ? error.message : String(error), }; } const external = plan.transactions.filter((transaction) => transaction.route === "external"); const previousSegments = new Map((previous?.segments ?? []).map((segment) => [ segment.segmentId, segment, ])); const segmentProgress = packageEvidence?.package.segments.map((segment) => { const persisted = previousSegments.get(segment.segmentId); if (persisted && ( persisted.handoffBlock !== segment.handoffBlock || JSON.stringify(persisted.transactionIds) !== JSON.stringify(segment.transactionIds) )) { throw new Error(`Persisted reconciliation segment ${segment.segmentId} changed`); } return { segmentId: segment.segmentId, transactionIds: segment.transactionIds, handoffBlock: segment.handoffBlock, nextBlock: persisted?.nextBlock ?? segment.handoffBlock, }; }); if (previousSegments.size > (segmentProgress?.length ?? 0)) { throw new Error("External execution package removed a reconciled segment"); } const firstIncompleteBeforeScan = plan.transactions.find((transaction) => { const match = matches.get(transaction.transactionId); return !match || match.receiptStatus !== 1 || !match.finalized; }); const activeSegment = firstIncompleteBeforeScan?.route === "external" ? packageEvidence?.package.segments.find((segment) => ( segment.transactionIds.includes(firstIncompleteBeforeScan.transactionId) )) : undefined; const activeProgress = activeSegment ? segmentProgress?.find((segment) => segment.segmentId === activeSegment.segmentId) : undefined; const writeScanCheckpoint = (): void => { const now = new Date().toISOString(); const checkpointMatches = [...matches.values()].sort((left, right) => ( left.transactionIndex - right.transactionIndex )); const checkpoint: ExternalReconciliationReport = { version: 1, status: "waiting_external", taskId: input.taskId, target: input.network, executionId: input.executionId, executionPlanHash: plan.planHash, packagePath: packageEvidence ? relative(input.root, packageEvidence.filePath) : undefined, packageHash: packageEvidence?.hash, segments: segmentProgress, finalizedBlock, finalityPolicy: finality.policy, matchedTransactionIds: checkpointMatches.map((match) => match.transactionId), matches: checkpointMatches, reportPath, createdAt: previous?.createdAt || now, updatedAt: now, }; writeJsonAtomic(reportFile, checkpoint); }; if ( !failure && activeSegment && activeProgress && !matches.has(firstIncompleteBeforeScan!.transactionId) ) { const latest = await provider.getBlockNumber(); const batchSize = input.batchSize ?? 1000; if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 1000) { throw new Error("Reconciliation batchSize must be between 1 and 1000"); } const start = activeProgress.nextBlock; const end = Math.min(latest, start + batchSize - 1); const segmentTransactions = activeSegment.transactionIds.map((transactionId) => { const transaction = plan.transactions.find((candidate) => ( candidate.transactionId === transactionId )); if (!transaction || transaction.route !== "external") { throw new Error(`External segment transaction ${transactionId} is invalid`); } return transaction; }); for (let blockNumber = start; blockNumber <= end; blockNumber += 1) { let block: { transactions?: RpcTransaction[] } | null; try { block = await provider.send( "eth_getBlockByNumber", [`0x${blockNumber.toString(16)}`, true], ) as { transactions?: RpcTransaction[] } | null; } catch (error) { writeScanCheckpoint(); throw error; } try { for (const transaction of block?.transactions || []) { const remaining = segmentTransactions.filter((candidate) => ( !matches.has(candidate.transactionId) )); const expected = remaining[0]; if (!expected) break; const code = await provider.getCode(expected.executor); if (!code || code === "0x") { if (!transactionMatches(transaction, expected, expected.executor)) continue; const receipt = await provider.getTransactionReceipt(transaction.hash); if (!receipt) continue; await validateCanonicalReceipt(provider, receipt); matches.set(expected.transactionId, receiptMatch( expected, transaction, receipt, finalizedBlock, { executionKind: "eoa", submittedBy: transaction.from }, )); continue; } const decoded = await safeCalls(provider, transaction, expected.executor); if (decoded.length === 0) continue; const receipt = await provider.getTransactionReceipt(transaction.hash); if (!receipt) continue; await validateCanonicalReceipt(provider, receipt); const matchedCalls: Array<{ transaction: ReleaseExecutionTransaction; decodedCall: (typeof decoded)[number]; }> = []; const remainingCalls = segmentTransactions.filter((item) => ( !matches.has(item.transactionId) )); for (const decodedCall of decoded) { const candidate = remainingCalls[matchedCalls.length]; if (!candidate) break; if ( decodedCall.to !== candidate.to || decodedCall.value !== candidate.value || decodedCall.data !== candidate.data || decodedCall.operation !== candidate.operation || candidate.executor !== expected.executor ) { break; } matchedCalls.push({ transaction: candidate, decodedCall }); } if (matchedCalls.length === 0) continue; assertSafeExecutionSucceeded(receipt, expected.executor); for (const { transaction: candidate, decodedCall } of matchedCalls) { matches.set(candidate.transactionId, receiptMatch( candidate, transaction, receipt, finalizedBlock, { executionKind: decodedCall.kind, submittedBy: transaction.from, innerCallIndex: decodedCall.innerCallIndex, }, )); } } activeProgress.nextBlock = blockNumber + 1; writeScanCheckpoint(); } catch (error) { failure = { code: "SCAN_FAILED", message: error instanceof Error ? error.message : String(error), }; break; } } } const orderedMatches = [...matches.values()].sort((left, right) => ( left.transactionIndex - right.transactionIndex )); const reverted = orderedMatches.find((match) => match.receiptStatus === 0); if (reverted) { failure = { code: "TRANSACTION_REVERTED", message: `Transaction ${reverted.transactionHash} reverted`, transactionId: reverted.transactionId, }; } const chainOrdered = [...orderedMatches].sort((left, right) => ( left.blockNumber - right.blockNumber || left.blockTransactionIndex - right.blockTransactionIndex || (left.innerCallIndex ?? -1) - (right.innerCallIndex ?? -1) )); if (chainOrdered.some((match, index) => ( index > 0 && chainOrdered[index - 1].transactionIndex >= match.transactionIndex ))) { failure = { code: "PLAN_ORDER_VIOLATION", message: "Canonical chain execution does not preserve immutable plan order", }; } const firstIncomplete = plan.transactions.find((transaction) => { const match = matches.get(transaction.transactionId); return !match || match.receiptStatus !== 1 || !match.finalized; }); const incompleteMatch = firstIncomplete ? matches.get(firstIncomplete.transactionId) : undefined; const submittedOperatorIds = new Set( (operatorResult?.receipts ?? []).map((receipt) => receipt.transactionId), ); const packageContainsIncomplete = Boolean( firstIncomplete?.route === "external" && packageEvidence?.package.segments.some((segment) => ( segment.transactionIds.includes(firstIncomplete.transactionId) )), ); const status: ExternalReconciliationReport["status"] = failure ? "failed" : !firstIncomplete ? external.length > 0 ? "reconciled" : "not_required" : incompleteMatch ? "waiting_finality" : firstIncomplete.route === "operator" ? submittedOperatorIds.has(firstIncomplete.transactionId) ? "waiting_finality" : "waiting_operator" : packageContainsIncomplete ? "waiting_external" : "waiting_operator"; const now = new Date().toISOString(); const report: ExternalReconciliationReport = { version: 1, status, taskId: input.taskId, target: input.network, executionId: input.executionId, executionPlanHash: plan.planHash, packagePath: packageEvidence ? relative(input.root, packageEvidence.filePath) : undefined, packageHash: packageEvidence?.hash, segments: segmentProgress, finalizedBlock, finalityPolicy: finality.policy, matchedTransactionIds: orderedMatches.map((match) => match.transactionId), matches: orderedMatches, failure, reportPath, createdAt: previous?.createdAt || now, updatedAt: now, completedAt: status === "reconciled" || status === "not_required" ? now : undefined, }; writeJsonAtomic(reportFile, report); if (status === "reconciled") { markDeploymentRecordReconciled({ taskDir, taskId: input.taskId, network: input.network, executionId: input.executionId, reportPath, reportHash: sha256(fs.readFileSync(reportFile)), }); } return report; };