import crypto, { type UUID } from "node:crypto"; import url from "node:url"; import type { Common } from "@/internal/common.js"; import type { Logger } from "@/internal/logger.js"; import type { Chain, SyncBlock, SyncBlockHeader } from "@/internal/types.js"; import { eth_getBlockByNumber, standardizeBlock } from "@/rpc/actions.js"; import { createQueue } from "@/utils/queue.js"; import { startClock } from "@/utils/timer.js"; import { wait } from "@/utils/wait.js"; import { type GetLogsRetryHelperParameters, getLogsRetryHelper, } from "@ponder/utils"; import { BlockNotFoundError, type EIP1193Parameters, type EIP1193RequestFn, type Hash, HttpRequestError, JsonRpcVersionUnsupportedError, MethodNotFoundRpcError, MethodNotSupportedRpcError, ParseRpcError, type PublicRpcSchema, type RpcError, type RpcTransactionReceipt, TimeoutError, custom, hexToNumber, isHex, webSocket, } from "viem"; import { WebSocket } from "ws"; import type { DebugRpcSchema } from "../utils/debug.js"; import { getHttpRpcClient } from "./http.js"; export type RpcSchema = [ ...PublicRpcSchema, ...DebugRpcSchema, /** * @description Returns the receipts of a block specified by hash * * @example * provider.request({ method: 'eth_getBlockReceipts', params: ['0x...'] }) * // => [{ ... }, { ... }] */ { Method: "eth_getBlockReceipts"; Parameters: [hash: Hash]; ReturnType: RpcTransactionReceipt[] | null; }, ]; export type RequestParameters = EIP1193Parameters; export type RequestReturnType< method extends EIP1193Parameters["method"], > = Extract["ReturnType"]; export type Rpc = { hostnames: string[]; request: ( parameters: TParameters, context?: { logger?: Logger; retryNullBlockRequest?: boolean }, ) => Promise>; subscribe: (params: { onBlock: (block: SyncBlock | SyncBlockHeader) => Promise; onError: (error: Error) => void; }) => void; unsubscribe: () => Promise; }; const RETRY_COUNT = 9; const BASE_DURATION = 125; const INITIAL_REACTIVATION_DELAY = 100; const MAX_REACTIVATION_DELAY = 5_000; const BACKOFF_FACTOR = 1.5; const LATENCY_WINDOW_SIZE = 500; /** Hurdle rate for switching to a faster bucket. */ const LATENCY_HURDLE_RATE = 0.1; /** Exploration rate. */ const EPSILON = 0.1; const INITIAL_MAX_RPS = 20; const MIN_RPS = 3; const MAX_RPS = 500; const RPS_INCREASE_FACTOR = 1.05; const RPS_DECREASE_FACTOR = 0.95; const RPS_INCREASE_QUALIFIER = 0.9; const SUCCESS_MULTIPLIER = 5; type Bucket = { index: number; hostname: string; /** Reactivation delay in milliseconds. */ reactivationDelay: number; /** Number of active connections. */ activeConnections: number; /** Is the bucket available to send requests. */ isActive: boolean; /** Is the bucket recently activated and yet to complete successful requests. */ isWarmingUp: boolean; latencyMetadata: { latencies: { value: number; success: boolean }[]; successfulLatencies: number; latencySum: number; }; expectedLatency: number; rps: { count: number; timestamp: number }[]; /** Number of consecutive successful requests. */ consecutiveSuccessfulRequests: number; /** Maximum requests per second (dynamic). */ rpsLimit: number; request: EIP1193RequestFn; }; const addLatency = (bucket: Bucket, latency: number, success: boolean) => { bucket.latencyMetadata.latencies.push({ value: latency, success }); bucket.latencyMetadata.latencySum += latency; if (success) { bucket.latencyMetadata.successfulLatencies++; } if (bucket.latencyMetadata.latencies.length > LATENCY_WINDOW_SIZE) { const record = bucket.latencyMetadata.latencies.shift()!; bucket.latencyMetadata.latencySum -= record.value; if (record.success) { bucket.latencyMetadata.successfulLatencies--; } } bucket.expectedLatency = bucket.latencyMetadata.latencySum / bucket.latencyMetadata.successfulLatencies; }; /** * Return `true` if the bucket is available to send a request. */ const isAvailable = (bucket: Bucket) => { if (bucket.isActive === false) return false; const now = Math.floor(Date.now() / 1000); const currentRPS = bucket.rps.find((r) => r.timestamp === now); if (currentRPS && currentRPS.count + 1 > bucket.rpsLimit) { return false; } if (bucket.rps.length > 0 && bucket.rps[0]!.timestamp < now) { const elapsed = now - bucket.rps[0]!.timestamp; const totalCount = bucket.rps.reduce((acc, rps) => acc + rps.count, 0); if (totalCount > bucket.rpsLimit * (1 + elapsed)) { return false; } } if (bucket.isWarmingUp && bucket.activeConnections > 3) { return false; } return true; }; export const createRpc = ({ common, chain, concurrency = 25, }: { common: Common; chain: Chain; concurrency?: number }): Rpc => { let backends: { request: EIP1193RequestFn; hostname: string }[]; let requestId: UUID | undefined; if (typeof chain.rpc === "string") { const protocol = new url.URL(chain.rpc).protocol; const hostname = new url.URL(chain.rpc).hostname; if (protocol === "https:" || protocol === "http:") { const httpRpcClient = getHttpRpcClient(chain.rpc, { common, chain, timeout: 10_000, }); backends = [ { request: custom({ request(body) { if (requestId) { return httpRpcClient.request({ body, fetchOptions: { headers: { "X-Request-ID": requestId } }, }); } return httpRpcClient.request({ body }); }, })({ chain: chain.viemChain, retryCount: 0, }).request, hostname, }, ]; } else if (protocol === "wss:" || protocol === "ws:") { backends = [ { request: webSocket(chain.rpc)({ chain: chain.viemChain, retryCount: 0, timeout: 10_000, }).request, hostname, }, ]; } else { throw new Error(`Unsupported RPC URL protocol: ${protocol}`); } } else if (Array.isArray(chain.rpc)) { backends = chain.rpc.map((rpc) => { const protocol = new url.URL(rpc).protocol; const hostname = new url.URL(chain.rpc).hostname; if (protocol === "https:" || protocol === "http:") { const httpRpcClient = getHttpRpcClient(rpc, { common, chain, timeout: 10_000, }); return { request: custom({ request(body) { if (requestId) { return httpRpcClient.request({ body, fetchOptions: { headers: { "X-Request-ID": requestId } }, }); } return httpRpcClient.request({ body }); }, })({ chain: chain.viemChain, retryCount: 0, }).request, hostname, }; } else if (protocol === "wss:" || protocol === "ws:") { return { request: webSocket(rpc)({ chain: chain.viemChain, retryCount: 0, timeout: 10_000, }).request, hostname, }; } else { throw new Error(`Unsupported RPC URL protocol: ${protocol}`); } }); } else { backends = [ { request: chain.rpc({ chain: chain.viemChain, retryCount: 0, timeout: 10_000, }).request, hostname: "custom_transport", }, ]; } if (typeof chain.ws === "string") { const protocol = new url.URL(chain.ws).protocol; if (protocol !== "wss:" && protocol !== "ws:") { throw new Error( `Inconsistent RPC URL protocol: ${protocol}. Expected wss or ws.`, ); } } const buckets: Bucket[] = backends.map( ({ request, hostname }, index) => ({ index, hostname, reactivationDelay: INITIAL_REACTIVATION_DELAY, activeConnections: 0, isActive: true, isWarmingUp: false, latencyMetadata: { latencies: [], successfulLatencies: 0, latencySum: 0, }, expectedLatency: 200, rps: [], consecutiveSuccessfulRequests: 0, rpsLimit: INITIAL_MAX_RPS, request, }) satisfies Bucket, ); let noAvailableBucketsTimer: NodeJS.Timeout | undefined; /** Tracks all active bucket reactivation timeouts to cleanup during shutdown */ const timeouts = new Set(); const scheduleBucketActivation = (bucket: Bucket) => { const delay = bucket.reactivationDelay; const timeoutId = setTimeout(() => { bucket.isActive = true; bucket.isWarmingUp = true; timeouts.delete(timeoutId); common.logger.debug({ msg: "JSON-RPC provider reactivated after rate limiting", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, retry_delay: Math.round(delay), }); }, delay); common.logger.debug({ msg: "JSON-RPC provider deactivated due to rate limiting", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, retry_delay: Math.round(delay), }); timeouts.add(timeoutId); }; const getBucket = async (): Promise => { let availableBuckets: Bucket[]; // Note: wait for the next event loop to ensure that the bucket rps are updated await new Promise((resolve) => setImmediate(resolve)); while (true) { // Remove old request per second data const timestamp = Math.floor(Date.now() / 1000); for (const bucket of buckets) { bucket.rps = bucket.rps.filter((r) => r.timestamp > timestamp - 10); } availableBuckets = buckets.filter(isAvailable); if (availableBuckets.length > 0) { break; } if (noAvailableBucketsTimer === undefined) { noAvailableBucketsTimer = setTimeout(() => { common.logger.warn({ msg: "All JSON-RPC providers are inactive due to rate limiting", chain: chain.name, chain_id: chain.id, rate_limits: JSON.stringify(buckets.map((b) => b.rpsLimit)), }); }, 5_000); } await wait(20); } clearTimeout(noAvailableBucketsTimer); noAvailableBucketsTimer = undefined; if (Math.random() < EPSILON) { const randomBucket = availableBuckets[Math.floor(Math.random() * availableBuckets.length)]!; randomBucket.activeConnections++; return randomBucket; } const fastestBucket = availableBuckets.reduce((fastest, current) => { const currentLatency = current.expectedLatency; const fastestLatency = fastest.expectedLatency; if (currentLatency < fastestLatency * (1 - LATENCY_HURDLE_RATE)) { return current; } if ( currentLatency <= fastestLatency && current.activeConnections < fastest.activeConnections ) { return current; } return fastest; }, availableBuckets[0]!); fastestBucket.activeConnections++; return fastestBucket; }; const increaseMaxRPS = (bucket: Bucket) => { if (bucket.rps.length < 10) return; if ( bucket.consecutiveSuccessfulRequests < bucket.rpsLimit * SUCCESS_MULTIPLIER ) { return; } for (const { count } of bucket.rps) { if (count < bucket.rpsLimit * RPS_INCREASE_QUALIFIER) { return; } } bucket.rpsLimit = Math.min(bucket.rpsLimit * RPS_INCREASE_FACTOR, MAX_RPS); bucket.consecutiveSuccessfulRequests = 0; common.logger.debug({ msg: "Increased JSON-RPC provider RPS limit", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, rps_limit: Math.floor(bucket.rpsLimit), }); }; const queue = createQueue< Awaited>, { body: Parameters[0]; context?: Parameters[1]; } >({ initialStart: true, concurrency, worker: async ({ body, context }) => { const logger = context?.logger ?? common.logger; for (let i = 0; i <= RETRY_COUNT; i++) { let endClock = startClock(); const t = setTimeout(() => { logger.warn({ msg: "Unable to find available JSON-RPC provider within expected time", chain: chain.name, chain_id: chain.id, rate_limit: JSON.stringify(buckets.map((b) => b.rpsLimit)), is_active: JSON.stringify(buckets.map((b) => b.isActive)), is_warming_up: JSON.stringify(buckets.map((b) => b.isWarmingUp)), duration: 15_000, }); }, 15_000); const bucket = await getBucket(); clearTimeout(t); const getBucketDuration = endClock(); endClock = startClock(); const id = crypto.randomUUID(); const surpassTimeout = setTimeout(() => { logger.warn({ msg: "JSON-RPC request unexpectedly surpassed timeout", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, request_id: id, method: body.method, duration: 15_000, }); }, 15_000); try { logger.trace({ msg: "Sent JSON-RPC request", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, request_id: id, method: body.method, duration: getBucketDuration, }); // Add request per second data const timestamp = Math.floor(Date.now() / 1000); if ( bucket.rps.length === 0 || bucket.rps[bucket.rps.length - 1]!.timestamp < timestamp ) { bucket.rps.push({ count: 1, timestamp }); } else { bucket.rps[bucket.rps.length - 1]!.count++; } requestId = id; const response = await bucket.request(body); if (response === undefined) { throw new Error("Response is undefined"); } if ( response === null && (body.method === "eth_getBlockByNumber" || body.method === "eth_getBlockByHash") && context?.retryNullBlockRequest === true ) { // Note: Throwing this error will cause the request to be retried. throw new BlockNotFoundError({ // @ts-ignore blockNumber: body.params[0], }); } const duration = endClock(); logger.trace({ msg: "Received JSON-RPC response", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, request_id: id, method: body.method, duration, }); common.metrics.ponder_rpc_request_duration.observe( { method: body.method, chain: chain.name }, duration, ); addLatency(bucket, duration, true); bucket.consecutiveSuccessfulRequests++; increaseMaxRPS(bucket); bucket.isWarmingUp = false; bucket.reactivationDelay = INITIAL_REACTIVATION_DELAY; return response as RequestReturnType; } catch (e) { const error = e as Error; common.metrics.ponder_rpc_request_error_total.inc( { method: body.method, chain: chain.name }, 1, ); if ( body.method === "eth_getLogs" && isHex(body.params[0].fromBlock) && isHex(body.params[0].toBlock) && chain.ethGetLogsBlockRange === undefined ) { const getLogsErrorResponse = getLogsRetryHelper({ params: body.params as GetLogsRetryHelperParameters["params"], error: error as RpcError, }); if (getLogsErrorResponse.shouldRetry) { common.logger.trace({ msg: "Caught eth_getLogs range error", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, request_id: id, method: body.method, request: JSON.stringify(body), retry_ranges: JSON.stringify(getLogsErrorResponse.ranges), error: error as Error, }); throw error; } else { // @ts-ignore error.meta = [ "Tip: Use the ethGetLogsBlockRange option to override the default behavior for this chain", ]; } } addLatency(bucket, endClock(), false); if ( // @ts-ignore error.code === 429 || // @ts-ignore error.status === 429 || error instanceof TimeoutError ) { if (bucket.isActive) { bucket.isActive = false; bucket.isWarmingUp = false; bucket.rpsLimit = Math.max( bucket.rpsLimit * RPS_DECREASE_FACTOR, MIN_RPS, ); bucket.consecutiveSuccessfulRequests = 0; common.logger.debug({ msg: "JSON-RPC provider rate limited", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, rps_limit: Math.floor(bucket.rpsLimit), }); scheduleBucketActivation(bucket); if (buckets.every((b) => b.isActive === false)) { logger.warn({ msg: "All JSON-RPC providers are inactive", chain: chain.name, chain_id: chain.id, }); } bucket.reactivationDelay = error instanceof TimeoutError ? INITIAL_REACTIVATION_DELAY : Math.min( bucket.reactivationDelay * BACKOFF_FACTOR, MAX_REACTIVATION_DELAY, ); } } if (shouldRetry(error) === false) { logger.warn({ msg: "Received JSON-RPC error", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, request_id: id, method: body.method, request: JSON.stringify(body), duration: endClock(), error, }); throw error; } if (i === RETRY_COUNT) { logger.warn({ msg: "Received JSON-RPC error", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, request_id: id, method: body.method, request: JSON.stringify(body), duration: endClock(), retry_count: i + 1, error, }); throw error; } const duration = BASE_DURATION * 2 ** i; logger.warn({ msg: "Received JSON-RPC error", chain: chain.name, chain_id: chain.id, hostname: bucket.hostname, request_id: id, method: body.method, request: JSON.stringify(body), duration: endClock(), retry_count: i + 1, retry_delay: duration, error, }); await wait(duration); } finally { bucket.activeConnections--; clearTimeout(surpassTimeout); } } throw "unreachable"; }, }); let ws: WebSocket | undefined; let isUnsubscribed = false; let subscriptionId: string | undefined; let webSocketErrorCount = 0; let interval: NodeJS.Timeout | undefined; const rpc: Rpc = { hostnames: backends.map((backend) => backend.hostname), // @ts-ignore request: (parameters, context) => queue.add({ body: parameters, context }), subscribe({ onBlock, onError }) { (async () => { while (true) { if (isUnsubscribed) return; if (chain.ws === undefined || webSocketErrorCount >= RETRY_COUNT) { const blockTag = chain.readPending ? "pending" : "latest"; common.logger.debug({ msg: "Created JSON-RPC polling subscription", chain: chain.name, chain_id: chain.id, polling_interval: chain.pollingInterval, block_tag: blockTag, }); interval = setInterval(async () => { try { const block = await eth_getBlockByNumber(rpc, [blockTag, true]); common.logger.trace({ msg: "Received successful JSON-RPC polling response", chain: chain.name, chain_id: chain.id, block_number: hexToNumber(block.number), block_hash: block.hash, }); // Note: `onBlock` should never throw. await onBlock(block); } catch (error) { onError(error as Error); } }, chain.pollingInterval); common.shutdown.add(() => { clearInterval(interval); }); return; } await new Promise((resolve) => { ws = new WebSocket(chain.ws!); ws.on("open", () => { common.logger.debug({ msg: "Created JSON-RPC WebSocket connection", chain: chain.name, chain_id: chain.id, }); const subscriptionRequest = { jsonrpc: "2.0", id: 1, method: "eth_subscribe", params: ["newHeads"], }; ws?.send(JSON.stringify(subscriptionRequest)); }); ws.on("message", (data: Buffer) => { try { const msg = JSON.parse(data.toString()); if ( msg.method === "eth_subscription" && msg.params.subscription === subscriptionId ) { common.logger.trace({ msg: "Received successful JSON-RPC WebSocket subscription data", chain: chain.name, chain_id: chain.id, block_number: msg.params.result.number ? hexToNumber(msg.params.result.number) : undefined, block_hash: msg.params.result.hash, }); webSocketErrorCount = 0; onBlock( standardizeBlock(msg.params.result, { method: "eth_subscribe", params: ["newHeads"], }), ); } else if (msg.result) { common.logger.debug({ msg: "Created JSON-RPC WebSocket subscription", chain: chain.name, chain_id: chain.id, request: JSON.stringify({ method: "eth_subscribe", params: ["newHeads"], }), subscription: msg.result, }); subscriptionId = msg.result; } else if (msg.error) { common.logger.warn({ msg: "Failed JSON-RPC WebSocket subscription", chain: chain.name, chain_id: chain.id, request: JSON.stringify({ method: "eth_subscribe", params: ["newHeads"], }), retry_count: webSocketErrorCount + 1, error: msg.error as Error, }); if (webSocketErrorCount < RETRY_COUNT) { webSocketErrorCount += 1; } ws?.close(); } else { common.logger.warn({ msg: "Received unrecognized JSON-RPC WebSocket message", chain: chain.name, websocket_message: msg, }); } } catch (error) { common.logger.warn({ msg: "Failed JSON-RPC WebSocket subscription", chain: chain.name, chain_id: chain.id, request: JSON.stringify({ method: "eth_subscribe", params: ["newHeads"], }), retry_count: webSocketErrorCount + 1, error: error as Error, }); if (webSocketErrorCount < RETRY_COUNT) { webSocketErrorCount += 1; } ws?.close(); } }); ws.on("error", async (error) => { common.logger.warn({ msg: "Failed JSON-RPC WebSocket subscription", chain: chain.name, chain_id: chain.id, request: JSON.stringify({ method: "eth_subscribe", params: ["newHeads"], }), retry_count: webSocketErrorCount + 1, error: error as Error, }); if (webSocketErrorCount < RETRY_COUNT) { webSocketErrorCount += 1; } if (ws && ws.readyState === ws.OPEN) { ws.close(); } else { resolve(); } }); ws.on("close", async () => { common.logger.debug({ msg: "Closed JSON-RPC WebSocket connection", chain: chain.name, chain_id: chain.id, }); ws = undefined; if (isUnsubscribed || webSocketErrorCount >= RETRY_COUNT) { resolve(); } else { const duration = BASE_DURATION * 2 ** webSocketErrorCount; common.logger.debug({ msg: "Retrying JSON-RPC WebSocket connection", chain: chain.name, retry_count: webSocketErrorCount + 1, retry_delay: duration, }); await wait(duration); resolve(); } }); }); } })(); }, async unsubscribe() { clearInterval(interval); isUnsubscribed = true; if (ws) { if (subscriptionId) { const unsubscribeRequest = { jsonrpc: "2.0", id: 1, method: "eth_unsubscribe", params: [subscriptionId], }; common.logger.debug({ msg: "Ended JSON-RPC WebSocket subscription", chain: chain.name, chain_id: chain.id, request: JSON.stringify({ method: "eth_unsubscribe", params: [subscriptionId], }), }); ws.send(JSON.stringify(unsubscribeRequest)); } ws.close(); } }, }; common.shutdown.add(() => { for (const timeoutId of timeouts) { clearTimeout(timeoutId); } timeouts.clear(); }); return rpc; }; /** * @link https://github.com/wevm/viem/blob/main/src/utils/buildtask.ts#L192 */ function shouldRetry(error: Error) { if ("code" in error && typeof error.code === "number") { // Invalid JSON if (error.code === ParseRpcError.code) return false; // Method does not exist if (error.code === MethodNotFoundRpcError.code) return false; // Method is not implemented if (error.code === MethodNotSupportedRpcError.code) return false; // Version of JSON-RPC protocol is not supported if (error.code === JsonRpcVersionUnsupportedError.code) return false; // eth_call reverted if (error.message.includes("revert")) return false; } if (error instanceof HttpRequestError && error.status) { // Method Not Allowed if (error.status === 405) return false; // Not Found if (error.status === 404) return false; // Not Implemented if (error.status === 501) return false; // HTTP Version Not Supported if (error.status === 505) return false; } return true; }