import * as Metrics from '../Metrics.js' import type * as Webhooks from '../Webhooks.js' import type * as Log from './Log.js' import * as MetricSink from './MetricSink.js' type MetricEvent = { name: string tags: Metrics.Tags | undefined type: 'count' | 'gauge' | 'histogram' value: number } describe('requests', () => { test('records optional funding count failures on successful pages', () => { const events: MetricEvent[] = [] const metrics = Metrics.from({ count(name, value, tags) { events.push({ name, tags, type: 'count', value }) }, flush() {}, gauge() {}, histogram() {}, }) const record = MetricSink.requests(metrics) record({ duration: 10, fundingDepositCountFailed: true, fundingTransferCountFailed: true, level: 'info', method: 'GET', path: '/v1/funding/transfers', requestId: 'request_private', route: '/v1/funding/transfers', status: 200, }) expect(events.find((event) => event.name === 'funding_deposit_count_failure_count')) .toMatchInlineSnapshot(` { "name": "funding_deposit_count_failure_count", "tags": { "cache": "miss", "method": "GET", "principal_environment": "none", "principal_type": "unknown", "route": "/v1/funding/transfers", }, "type": "count", "value": 1, } `) expect(events.find((event) => event.name === 'funding_transfer_count_failure_count')) .toMatchInlineSnapshot(` { "name": "funding_transfer_count_failure_count", "tags": { "cache": "miss", "method": "GET", "principal_environment": "none", "principal_type": "unknown", "route": "/v1/funding/transfers", }, "type": "count", "value": 1, } `) }) test('records bounded operational failure signals', () => { const events: MetricEvent[] = [] const metrics = Metrics.from({ count(name, value, tags) { events.push({ name, tags, type: 'count', value }) }, flush() {}, gauge() {}, histogram() {}, }) const record = MetricSink.requests(metrics) record({ duration: 10, errorCode: 'upstream_error', level: 'error', method: 'POST', path: '/rpc/sponsor', providerFailures: [ { chainId: 4217, failure: 'http', id: 'tidx', operation: 'query', status: 502, }, ], requestId: 'request_private', route: '/rpc/sponsor', rpc: { code: -32603, dataCode: 'internal_error', errors: 2, serverErrors: 2 }, sponsorship: { chainId: 4217, internalErrors: 2, method: 'eth_sendRawTransaction', outcome: 'rejected', payloadHash: `0x${'aa'.repeat(32)}`, rejections: 2, }, status: 502, }) expect(events.filter((event) => event.name !== 'http_response_count')).toMatchInlineSnapshot(` [ { "name": "http_server_error_count", "tags": { "cache": "miss", "error_code": "upstream_error", "method": "POST", "principal_environment": "none", "principal_type": "unknown", "route": "/rpc/sponsor", "status": 502, }, "type": "count", "value": 1, }, { "name": "rpc_response_error_count", "tags": { "cache": "miss", "method": "POST", "principal_environment": "none", "principal_type": "unknown", "route": "/rpc/sponsor", "rpc_error_code": "-32603", }, "type": "count", "value": 2, }, { "name": "rpc_response_server_error_count", "tags": { "cache": "miss", "method": "POST", "principal_environment": "none", "principal_type": "unknown", "route": "/rpc/sponsor", "rpc_error_code": "-32603", }, "type": "count", "value": 2, }, { "name": "upstream_failure_count", "tags": { "cache": "miss", "chain_id": "4217", "failure": "http", "method": "POST", "operation": "query", "principal_environment": "none", "principal_type": "unknown", "provider": "tidx", "route": "/rpc/sponsor", "status": "502", }, "type": "count", "value": 1, }, { "name": "sponsorship_internal_error_count", "tags": { "cache": "miss", "chain_id": "4217", "method": "eth_sendRawTransaction", "principal_environment": "none", "principal_type": "unknown", "route": "/rpc/sponsor", }, "type": "count", "value": 2, }, ] `) expect(JSON.stringify(events)).not.toContain('request_private') expect(JSON.stringify(events)).not.toContain('aa'.repeat(32)) }) test('records JSON-RPC errors with bounded code tags', () => { const events: MetricEvent[] = [] const metrics = Metrics.from({ count(name, value, tags) { events.push({ name, tags, type: 'count', value }) }, flush() {}, gauge() {}, histogram() {}, }) const record = MetricSink.requests(metrics) record({ duration: 10, level: 'warn', method: 'POST', path: '/rpc/relay', requestId: 'request_private', route: '/rpc/relay', rpc: { code: -32602, dataCode: 'billing_required', errors: 2 }, status: 200, }) expect(events).toMatchInlineSnapshot(` [ { "name": "http_response_count", "tags": { "cache": "miss", "method": "POST", "principal_environment": "none", "principal_type": "unknown", "route": "/rpc/relay", "status": 200, }, "type": "count", "value": 1, }, { "name": "rpc_response_error_count", "tags": { "cache": "miss", "method": "POST", "principal_environment": "none", "principal_type": "unknown", "route": "/rpc/relay", "rpc_error_code": "-32602", }, "type": "count", "value": 2, }, ] `) }) test('records bounded funding provider attempts without caller data', () => { const events: MetricEvent[] = [] let flushes = 0 const metrics = Metrics.from({ count(name, value, tags) { events.push({ name, tags, type: 'count', value }) }, flush() { flushes += 1 }, gauge(name, value, tags) { events.push({ name, tags, type: 'gauge', value }) }, histogram(name, value, tags) { events.push({ name, tags, type: 'histogram', value }) }, }) const record = MetricSink.requests(metrics) record({ duration: 100, fundingProviderAttempts: [ { durationMs: 10, id: 'relay', operation: 'getQuote', outcome: 'available', }, { durationMs: 20, id: 'rhino', operation: 'getQuote', outcome: 'thin', }, { durationMs: 30, id: 'squid', operation: 'getQuote', outcome: 'unavailable', }, { durationMs: 40, failure: 'timeout', id: 'across', operation: 'prepareTransfer', outcome: 'failed', }, ], level: 'info', method: 'GET', path: '/v1/funding/quotes', principal: { id: 'key_private', orgId: 'org_private', projectId: 'project_private', type: 'api_key', }, query: 'sourceAmount=secret', requestId: 'request_private', route: '/v1/funding/quotes', status: 200, } satisfies Log.Entry) const funding = events.filter((event) => event.name.startsWith('funding_quote_provider_')) expect(funding).toMatchInlineSnapshot(` [ { "name": "funding_quote_provider_attempt_count", "tags": { "operation": "getQuote", "outcome": "available", "provider": "relay", }, "type": "count", "value": 1, }, { "name": "funding_quote_provider_duration_ms", "tags": { "operation": "getQuote", "outcome": "available", "provider": "relay", }, "type": "histogram", "value": 10, }, { "name": "funding_quote_provider_attempt_count", "tags": { "operation": "getQuote", "outcome": "thin", "provider": "rhino", }, "type": "count", "value": 1, }, { "name": "funding_quote_provider_duration_ms", "tags": { "operation": "getQuote", "outcome": "thin", "provider": "rhino", }, "type": "histogram", "value": 20, }, { "name": "funding_quote_provider_attempt_count", "tags": { "operation": "getQuote", "outcome": "unavailable", "provider": "squid", }, "type": "count", "value": 1, }, { "name": "funding_quote_provider_duration_ms", "tags": { "operation": "getQuote", "outcome": "unavailable", "provider": "squid", }, "type": "histogram", "value": 30, }, { "name": "funding_quote_provider_attempt_count", "tags": { "failure": "timeout", "operation": "prepareTransfer", "outcome": "failed", "provider": "across", }, "type": "count", "value": 1, }, { "name": "funding_quote_provider_duration_ms", "tags": { "failure": "timeout", "operation": "prepareTransfer", "outcome": "failed", "provider": "across", }, "type": "histogram", "value": 40, }, ] `) expect(JSON.stringify(funding)).not.toMatch( /key_private|org_private|project_private|request_private|secret/, ) expect(flushes).toBe(1) }) test('records MPP relay outcomes without credential data', () => { const events: MetricEvent[] = [] const metrics = Metrics.from({ count(name, value, tags) { events.push({ name, tags, type: 'count', value }) }, flush() {}, gauge() {}, histogram(name, value, tags) { events.push({ name, tags, type: 'histogram', value }) }, }) const record = MetricSink.requests(metrics) record({ duration: 42, level: 'info', method: 'POST', mpp: { chainId: 4217, errorCode: 'invalid_payment', feePayer: true, idempotency: 'claimed', operation: 'broadcast', outcome: 'failure', }, path: '/v1/mpp/broadcast', requestId: 'request_private', route: '/v1/mpp/broadcast', status: 200, } satisfies Log.Entry) const mpp = events.filter((event) => event.name.startsWith('mpp_relay_')) expect(mpp).toMatchInlineSnapshot(` [ { "name": "mpp_relay_operation_count", "tags": { "chain_id": "4217", "error_code": "invalid_payment", "fee_payer": true, "operation": "broadcast", "outcome": "failure", }, "type": "count", "value": 1, }, { "name": "mpp_relay_operation_duration_ms", "tags": { "chain_id": "4217", "error_code": "invalid_payment", "fee_payer": true, "operation": "broadcast", "outcome": "failure", }, "type": "histogram", "value": 42, }, { "name": "mpp_relay_idempotency_claim_count", "tags": { "chain_id": "4217", "outcome": "claimed", }, "type": "count", "value": 1, }, ] `) expect(JSON.stringify(mpp)).not.toMatch(/request_private|credential|payload|signature/) }) }) describe('webhooks', () => { test('records scan pipeline blocks, backlog, and finality lag', () => { const events: MetricEvent[] = [] const metrics = Metrics.from({ count(name, value, tags) { events.push({ name, tags, type: 'count', value }) }, flush() {}, gauge(name, value, tags) { events.push({ name, tags, type: 'gauge', value }) }, histogram() {}, }) const sink = MetricSink.webhooks(metrics) sink.record({ chainId: 42431, deferred: false, matches: 3, origin: 'socket', outcome: 'dispatched', type: 'webhook:scan-block', workMs: 12 }) // prettier-ignore sink.record({ chainId: 42431, deferred: true, matches: 0, origin: 'poll', outcome: 'deferred', type: 'webhook:scan-block', workMs: 3 }) // prettier-ignore sink.record({ chainId: 42431, oldestPendingAgeMs: 5_000, pendingCount: 2, type: 'webhook:scan-backlog', watermarkLagBlocks: 7 }) // prettier-ignore sink.record({ chainId: 42431, lagBlocks: 1, type: 'webhook:scan-finality' }) expect(events.map(({ name, tags, value }) => `${name}=${value} ${JSON.stringify(tags)}`)) .toMatchInlineSnapshot(` [ "webhook_scan_block_count=1 {"chain_id":"42431","origin":"socket","outcome":"dispatched"}", "webhook_scan_match_count=3 {"chain_id":"42431","origin":"socket","outcome":"dispatched"}", "webhook_scan_block_count=1 {"chain_id":"42431","origin":"poll","outcome":"deferred"}", "webhook_scan_pending_count=2 {"chain_id":"42431"}", "webhook_scan_oldest_pending_age_ms=5000 {"chain_id":"42431"}", "webhook_scan_watermark_lag_blocks=7 {"chain_id":"42431"}", "webhook_scan_finality_lag_blocks=1 {"chain_id":"42431"}", ] `) }) test('records scan timing spans and per-step attribution', () => { const events: MetricEvent[] = [] const metrics = Metrics.from({ count(name, value, tags) { events.push({ name, tags, type: 'count', value }) }, flush() {}, gauge() {}, histogram(name, value, tags) { events.push({ name, tags, type: 'histogram', value }) }, }) const sink = MetricSink.webhooks(metrics) sink.record({ chainId: 42431, deferred: false, matches: 1, observationLagMs: 600, origin: 'socket', outcome: 'dispatched', queueWaitMs: 900, scanLagMs: 1_500, type: 'webhook:scan-block', workMs: 400 }) // prettier-ignore // A replay carries no observation, so only the block-derived spans exist. sink.record({ chainId: 42431, deferred: false, matches: 0, origin: 'replay', outcome: 'empty', scanLagMs: 30_000_000, type: 'webhook:scan-block', workMs: 250 }) // prettier-ignore // Batch totals: the count is what makes 180ms across two calls readable // as a per-invocation mean rather than a slow single read. sink.record({ chainId: 42431, count: 2, durationMs: 180, step: 'finalized_rpc', type: 'webhook:scan-step' }) // prettier-ignore sink.record({ chainId: 42431, count: 1, durationMs: 95, step: 'subscription_read', type: 'webhook:scan-step' }) // prettier-ignore expect(events.map(({ name, tags, value }) => `${name}=${value} ${JSON.stringify(tags)}`)) .toMatchInlineSnapshot(` [ "webhook_scan_block_count=1 {"chain_id":"42431","origin":"socket","outcome":"dispatched"}", "webhook_scan_match_count=1 {"chain_id":"42431","origin":"socket","outcome":"dispatched"}", "webhook_scan_work_ms=400 {"chain_id":"42431","origin":"socket","outcome":"dispatched"}", "webhook_scan_queue_wait_ms=900 {"chain_id":"42431","origin":"socket","outcome":"dispatched","deferred":"false"}", "webhook_scan_observation_lag_ms=600 {"chain_id":"42431","origin":"socket"}", "webhook_scan_lag_ms=1500 {"chain_id":"42431"}", "webhook_scan_block_count=1 {"chain_id":"42431","origin":"replay","outcome":"empty"}", "webhook_scan_work_ms=250 {"chain_id":"42431","origin":"replay","outcome":"empty"}", "webhook_scan_lag_ms=30000000 {"chain_id":"42431"}", "webhook_scan_step_duration_ms=180 {"chain_id":"42431","step":"finalized_rpc"}", "webhook_scan_step_count=2 {"chain_id":"42431","step":"finalized_rpc"}", "webhook_scan_step_duration_ms=95 {"chain_id":"42431","step":"subscription_read"}", "webhook_scan_step_count=1 {"chain_id":"42431","step":"subscription_read"}", ] `) }) test('records obligation staging and claim outcomes', () => { const events: MetricEvent[] = [] const metrics = Metrics.from({ count(name, value, tags) { events.push({ name, tags, type: 'count', value }) }, flush() {}, gauge() {}, histogram() {}, }) const sink = MetricSink.webhooks(metrics) sink.record({ count: 3, outcome: 'created', type: 'webhook:delivery-job-ensure' }) sink.record({ count: 2, outcome: 'existing', type: 'webhook:delivery-job-ensure' }) sink.record({ outcome: 'claimed', type: 'webhook:delivery-job-claim' }) sink.record({ outcome: 'terminal', type: 'webhook:delivery-job-claim' }) expect(events).toMatchInlineSnapshot(` [ { "name": "webhook_delivery_job_ensure_count", "tags": { "outcome": "created", }, "type": "count", "value": 3, }, { "name": "webhook_delivery_job_ensure_count", "tags": { "outcome": "existing", }, "type": "count", "value": 2, }, { "name": "webhook_delivery_job_claim_count", "tags": { "outcome": "claimed", }, "type": "count", "value": 1, }, { "name": "webhook_delivery_job_claim_count", "tags": { "outcome": "terminal", }, "type": "count", "value": 1, }, ] `) }) test('records delivery stages without subscription identity', () => { const events: MetricEvent[] = [] const metrics = Metrics.from({ count(name, value, tags) { events.push({ name, tags, type: 'count', value }) }, flush() {}, gauge() {}, histogram(name, value, tags) { events.push({ name, tags, type: 'histogram', value }) }, }) const sink = MetricSink.webhooks(metrics) const subscription = { chainId: 4217, createdAt: '2026-01-01T00:00:00.000Z', destination: { type: 'url', url: 'https://hooks.example.com/endpoint' }, eventType: 'token:transfer', failureCount: 0, filters: {}, id: 'wh_private', owner: { orgId: 'org_private', type: 'api_key' }, secret: 'whsec_private', status: 'active', updatedAt: '2026-01-01T00:00:00.000Z', } satisfies Webhooks.Subscription const envelope = { chainId: 4217, createdAt: '2026-01-01T00:00:00.000Z', data: { ping: true }, id: 'evt_private', subscriptionId: subscription.id, type: 'ping', } satisfies Webhooks.Envelope sink.record({ attempt: 1, envelope, queueAttempt: 2, result: { durationMs: 25, ok: true, status: 204 }, subscription, timings: { deliveryLogMs: 5, envelopeToQueueMs: 3, endToEndMs: 250, eventToEnvelopeMs: 80, processingMs: 67, queueWaitMs: 100, stateWriteMs: 10, subscriptionReadMs: 4, }, trigger: 'queue', type: 'webhook:delivery', }) expect(events.map(({ name, value }) => ({ name, value }))).toMatchInlineSnapshot(` [ { "name": "webhook_delivery_attempt_count", "value": 1, }, { "name": "webhook_delivery_duration_ms", "value": 25, }, { "name": "webhook_delivery_log_write_ms", "value": 5, }, { "name": "webhook_delivery_envelope_to_queue_ms", "value": 3, }, { "name": "webhook_delivery_end_to_end_ms", "value": 250, }, { "name": "webhook_delivery_event_to_envelope_ms", "value": 80, }, { "name": "webhook_delivery_processing_ms", "value": 67, }, { "name": "webhook_delivery_queue_wait_ms", "value": 100, }, { "name": "webhook_delivery_state_write_ms", "value": 10, }, { "name": "webhook_delivery_subscription_read_ms", "value": 4, }, ] `) expect(events.find((event) => event.name === 'webhook_delivery_attempt_count')?.tags) .toMatchInlineSnapshot(` { "attempt_bucket": "1", "chain_id": "4217", "destination_type": "url", "event_type": "ping", "outcome": "succeeded", "queue_attempt_bucket": "2", "queue_disposition": "acked", "status_class": "2xx", "trigger": "queue", } `) expect(events.find((event) => event.name === 'webhook_delivery_queue_wait_ms')?.tags) .toMatchInlineSnapshot(` { "chain_id": "4217", "destination_type": "url", "event_type": "ping", "outcome": "succeeded", "queue_attempt_bucket": "2", "queue_disposition": "acked", } `) expect(JSON.stringify(events)).not.toMatch(/evt_private|org_private|wh_private|whsec_private/) sink.record({ attempt: 3, envelope, queueAttempt: 3, result: { error: 'upstream unavailable', ok: false, status: 503 }, subscription, trigger: 'queue', type: 'webhook:delivery', }) expect(events.at(-1)?.tags).toMatchObject({ queue_attempt_bucket: '3-5', queue_disposition: 'scheduled_retry', }) }) })