import type * as Log from '../internal/Log.js' import * as Analytics from './Analytics.js' import * as RequestEvents from './tables/requestEvents.js' const context = { enabled: true, environment: 'test', service: 'api' } const entry = { chainId: 4217, duration: 12.3, level: 'info', method: 'GET', path: '/v1/transactions', query: 'include=receipt&limit=10', requestId: 'req-1', route: '/v1/transactions', status: 200, timings: { transactions: 42.7, transactions_count: 7.1 }, } satisfies Log.Entry const clickHouse = { database: 'tempo_api', password: 'secret', url: 'https://clickhouse.test', user: 'reader', } /** Baseline stored row for insert/queue tests. */ const row = { billing_active: null, chain_id: 4217, duration_ms: 12.3, environment: 'test', error_code: null, key_environment: null, key_id: null, method: 'GET', org_id: null, project_id: null, principal_type: 'unknown', query: 'include=receipt&limit=10', rate_limit_scope: null, request_id: 'req-1', route: '/v1/transactions', rpc_error_code: null, rpc_error_count: 0, rpc_error_data_code: null, service: 'api', status: 200, timestamp: '2026-01-01 00:00:00.000', timings: { transactions: 42.7, transactions_count: 7.1 }, } satisfies Analytics.Event afterEach(() => vi.unstubAllGlobals()) describe('clickhouse', () => { test('insert posts NDJSON rows scoped to the database', async () => { const fetch = vi .fn() .mockResolvedValue(new Response('', { status: 200 })) vi.stubGlobal('fetch', fetch) await Analytics.clickhouse(clickHouse).insert('request_events', [row]) const url = new URL(String(fetch.mock.calls[0]![0])) expect(url.searchParams.get('database')).toBe('tempo_api') expect(url.searchParams.get('query')).toBe('INSERT INTO `request_events` FORMAT JSONEachRow') const init = fetch.mock.calls[0]![1] expect(init?.body).toBe(`${JSON.stringify(row)}\n`) expect(init?.headers).toMatchObject({ Authorization: `Basic ${btoa('reader:secret')}`, 'Content-Type': 'application/x-ndjson', }) }) test('insert skips empty batches', async () => { const fetch = vi.fn() vi.stubGlobal('fetch', fetch) await Analytics.clickhouse(clickHouse).insert('request_events', []) expect(fetch).not.toHaveBeenCalled() }) test('insert throws InsertError on a non-2xx response', async () => { vi.stubGlobal( 'fetch', vi .fn() .mockResolvedValue(new Response('table missing', { status: 404 })), ) await expect( Analytics.clickhouse(clickHouse).insert('request_events', [row]), ).rejects.toThrowErrorMatchingInlineSnapshot( `[Analytics.InsertError: ClickHouse insert failed with status 404: table missing]`, ) }) test('query posts the statement in the body and returns rows', async () => { const fetch = vi .fn() .mockResolvedValue(new Response(JSON.stringify({ data: [{ requests: 1 }] }), { status: 200 })) // prettier-ignore vi.stubGlobal('fetch', fetch) const rows = await Analytics.clickhouse(clickHouse).query('SELECT count() AS requests') expect(rows).toEqual([{ requests: 1 }]) const url = new URL(String(fetch.mock.calls[0]![0])) expect(url.searchParams.get('database')).toBe('tempo_api') expect(url.searchParams.get('query')).toBeNull() expect(String(fetch.mock.calls[0]![1]?.body)).toBe('SELECT count() AS requests FORMAT JSON') }) test('query throws QueryError on a non-2xx response', async () => { vi.stubGlobal( 'fetch', vi .fn() .mockResolvedValue(new Response('syntax error', { status: 400 })), ) await expect( Analytics.clickhouse(clickHouse).query('SELECT'), ).rejects.toThrowErrorMatchingInlineSnapshot( `[Analytics.QueryError: ClickHouse query failed with status 400: syntax error]`, ) }) test('rejects invalid table identifiers', async () => { vi.stubGlobal('fetch', vi.fn()) await expect( Analytics.clickhouse(clickHouse).insert('bad`name' as 'request_events', [row]), ).rejects.toThrowErrorMatchingInlineSnapshot( `[Analytics.ConfigError: Invalid ClickHouse identifier: bad\`name]`, ) }) test('migrate applies every migration when guards are unsatisfied', async () => { // Guards report their columns missing, so every migration's DDL applies. const fetch = vi.fn().mockImplementation(async (_url, init) => { if (String(init?.body).includes('system.columns')) return new Response(JSON.stringify({ data: [{ applied: 0 }] }), { status: 200 }) return new Response('', { status: 200 }) }) vi.stubGlobal('fetch', fetch) await Analytics.clickhouse(clickHouse).migrate() const url = new URL(String(fetch.mock.calls[0]![0])) expect(url.searchParams.get('database')).toBe('tempo_api') const bodies = fetch.mock.calls.map((call) => String(call[1]?.body)) for (const migration of Analytics.migrations) expect(bodies).toContain(migration.sql) }) test('migrate skips a guarded migration when its guard is satisfied', async () => { // Guards report their columns present (fresh DB from the baseline CREATE), // so the guarded ALTER is skipped, sparing a scoped user the ALTER grant. const fetch = vi.fn().mockImplementation(async (_url, init) => { if (String(init?.body).includes('system.columns')) return new Response(JSON.stringify({ data: [{ applied: 1 }] }), { status: 200 }) return new Response('', { status: 200 }) }) vi.stubGlobal('fetch', fetch) await Analytics.clickhouse(clickHouse).migrate() const bodies = fetch.mock.calls.map((call) => String(call[1]?.body)) for (const migration of Analytics.migrations.filter((migration) => migration.guard)) expect(bodies).not.toContain(migration.sql) }) test('migrate throws MigrateError on a non-2xx response', async () => { vi.stubGlobal( 'fetch', vi.fn().mockResolvedValue(new Response('denied', { status: 403 })), ) await expect(Analytics.clickhouse(clickHouse).migrate()).rejects.toThrow(Analytics.MigrateError) }) }) describe('migrations', () => { test('the migrations declare every request_events column', () => { // The baseline CREATE declares every column; later ALTERs re-patch tables // that predate a column (e.g. billing columns for existing production). const sql = Analytics.migrations.map((migration) => migration.sql).join('\n') for (const column of Object.keys(row)) expect(sql).toContain(column) expect(sql).toContain('timings Map(String, Float64)') expect(sql).toContain('query String') }) test('the migrations declare query benchmark history', () => { const sql = Analytics.migrations.map((migration) => migration.sql).join('\n') expect(sql).toContain('CREATE TABLE IF NOT EXISTS query_benchmark_results') expect(sql).toContain('ORDER BY (chain_id, query, metric, timestamp, run_id)') expect(sql).toContain('rpc_url String') expect(sql).toContain('tidx_url String') }) test('the migrations declare the endpoint latency view', () => { const sql = Analytics.migrations.map((migration) => migration.sql).join('\n') const latest = Analytics.migrations[Analytics.migrations.length - 1]! expect(sql).toContain('CREATE OR REPLACE VIEW endpoint_latency_results') expect(sql).toContain("'production' AS source") expect(sql).toContain("'benchmark' AS source") expect(sql).toContain("if(result.metric = 'benchmark', 'request', result.metric) AS metric") expect(sql).toContain("if(status >= 200 AND status < 300, 'success', 'error') AS outcome") expect(sql).toContain("WHERE result.metric != 'request'") expect(sql).toContain('arrayZip(mapKeys(timings), mapValues(timings))') expect(latest.name).toBe('0011_endpoint_latency_query') expect(latest.sql).toContain("environment = 'production' AND service = 'cadent-api'") expect(latest.sql).toContain("coalesce(request_id, '') AS request_id") expect(latest.sql).toContain('query AS variant') }) }) describe('get', () => { test('resolves factories and passes stores through', () => { const store = Analytics.clickhouse(clickHouse) expect(Analytics.get(store)).toBe(store) expect(Analytics.get(() => store)).toBe(store) }) }) describe('createQueueSink', () => { test('enqueues one row per entry', async () => { const send = vi.fn<(message: Analytics.Event) => Promise>().mockResolvedValue(undefined) const sink = Analytics.createQueueSink({ send })(context) await sink(entry) expect(send).toHaveBeenCalledTimes(1) const row = send.mock.calls[0]![0] expect({ billing_active: row.billing_active, chain_id: row.chain_id, duration_ms: row.duration_ms, environment: row.environment, error_code: row.error_code, key_environment: row.key_environment, key_id: row.key_id, method: row.method, org_id: row.org_id, principal_type: row.principal_type, project_id: row.project_id, query: row.query, rate_limit_scope: row.rate_limit_scope, request_id: row.request_id, route: row.route, rpc_error_code: row.rpc_error_code, rpc_error_count: row.rpc_error_count, rpc_error_data_code: row.rpc_error_data_code, service: row.service, status: row.status, timestamp: '', timings: row.timings, }).toMatchInlineSnapshot(` { "billing_active": null, "chain_id": 4217, "duration_ms": 12.3, "environment": "test", "error_code": null, "key_environment": null, "key_id": null, "method": "GET", "org_id": null, "principal_type": "unknown", "project_id": null, "query": "include=receipt&limit=10", "rate_limit_scope": null, "request_id": "req-1", "route": "/v1/transactions", "rpc_error_code": null, "rpc_error_count": 0, "rpc_error_data_code": null, "service": "api", "status": 200, "timestamp": "", "timings": { "transactions": 42.7, "transactions_count": 7.1, }, } `) }) test('stores an empty timing map when the request has no operation timings', async () => { const send = vi.fn<(message: Analytics.Event) => Promise>().mockResolvedValue(undefined) const sink = Analytics.createQueueSink({ send })(context) await sink({ ...entry, timings: undefined }) expect(send.mock.calls[0]![0].timings).toMatchInlineSnapshot(`{}`) }) test('stores an empty query when the request has no query string', async () => { const send = vi.fn<(message: Analytics.Event) => Promise>().mockResolvedValue(undefined) const sink = Analytics.createQueueSink({ send })(context) await sink({ ...entry, query: undefined }) expect(send.mock.calls[0]![0].query).toMatchInlineSnapshot(`""`) }) test('retries a failed send once with the same row', async () => { const send = vi .fn<(message: Analytics.Event) => Promise>() .mockRejectedValueOnce(new Error('Unknown Internal Error (15000)')) .mockResolvedValueOnce(undefined) const sink = Analytics.createQueueSink({ send })(context) await sink(entry) expect(send).toHaveBeenCalledTimes(2) expect(send.mock.calls[1]![0]).toBe(send.mock.calls[0]![0]) }) test('records API-key attribution for API-key principals', async () => { const send = vi.fn<(message: Analytics.Event) => Promise>().mockResolvedValue(undefined) const sink = Analytics.createQueueSink({ send })(context) await sink({ ...entry, principal: { billingActive: true, environment: 'production', id: 'key_1', orgId: 'org_1', projectId: 'prj_1', type: 'api_key', }, }) expect(send.mock.calls[0]![0]).toMatchObject({ billing_active: 1, key_environment: 'production', project_id: 'prj_1', }) }) test('records JSON-RPC error metadata', async () => { const send = vi.fn<(message: Analytics.Event) => Promise>().mockResolvedValue(undefined) const sink = Analytics.createQueueSink({ send })(context) await sink({ ...entry, level: 'warn', rpc: { code: -32602, dataCode: 'billing_required', errors: 2 }, }) expect(send.mock.calls[0]![0]).toMatchObject({ rpc_error_code: -32602, rpc_error_count: 2, rpc_error_data_code: 'billing_required', status: 200, }) }) test('rethrows when the retry also fails', async () => { const send = vi .fn<(message: Analytics.Event) => Promise>() .mockRejectedValue(new Error('Unknown Internal Error (15000)')) const sink = Analytics.createQueueSink({ send })(context) await expect(sink(entry)).rejects.toThrowErrorMatchingInlineSnapshot( `[Error: Unknown Internal Error (15000)]`, ) expect(send).toHaveBeenCalledTimes(2) }) test('drops entries when disabled', async () => { const send = vi.fn<(message: Analytics.Event) => Promise>() const sink = Analytics.createQueueSink({ send })({ ...context, enabled: false }) await sink(entry) expect(send).not.toHaveBeenCalled() }) }) describe('handleQueue', () => { /** Fake store and one queued message with ack/retry spies. */ function fixture() { const insert = vi.fn((_table: 'request_events', _rows: readonly Analytics.Event[]) => Promise.resolve(), ) const store = { insert, migrate: () => Promise.resolve(), query: () => Promise.resolve([]) } const message = { ack: vi.fn(), body: row, retry: vi.fn() } return { insert, message, store } } test('ignores batches from other queues', async () => { const { insert, message, store } = fixture() const handled = await Analytics.handleQueue(store, { messages: [message], queue: 'webhook-deliveries', }) expect(handled).toBe(false) expect(insert).not.toHaveBeenCalled() expect(message.ack).not.toHaveBeenCalled() }) test('matches an overridden queue name instead of the default', async () => { const { insert, message, store } = fixture() const handled = await Analytics.handleQueue( store, { messages: [message], queue: 'api-pr-7-request-analytics' }, { queue: 'api-pr-7-request-analytics' }, ) expect(handled).toBe(true) expect(insert).toHaveBeenCalledWith('request_events', [row]) const defaultNamed = await Analytics.handleQueue( store, { messages: [message], queue: Analytics.queueName }, { queue: 'api-pr-7-request-analytics' }, ) expect(defaultNamed).toBe(false) expect(insert).toHaveBeenCalledTimes(1) }) test('inserts the batch and acks each message', async () => { const { insert, message, store } = fixture() const onResult = vi.fn() const handled = await Analytics.handleQueue( store, { messages: [message], queue: Analytics.queueName, }, { onResult }, ) expect(handled).toBe(true) expect(insert).toHaveBeenCalledWith('request_events', [row]) expect(message.ack).toHaveBeenCalledTimes(1) expect(message.retry).not.toHaveBeenCalled() expect(onResult).toHaveBeenCalledWith({ messages: 1, outcome: 'acked' }) }) test('retries every message when the insert fails', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}) const { insert, message, store } = fixture() const onResult = vi.fn() insert.mockRejectedValueOnce(new Error('clickhouse down')) const handled = await Analytics.handleQueue( store, { messages: [message], queue: Analytics.queueName, }, { onResult }, ) expect(handled).toBe(true) expect(message.ack).not.toHaveBeenCalled() expect(message.retry).toHaveBeenCalledTimes(1) expect(onResult).toHaveBeenCalledWith({ cause: expect.objectContaining({ message: 'clickhouse down' }), messages: 1, outcome: 'retried', }) }) test('does not retry acknowledged messages when the result callback fails', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}) const { message, store } = fixture() const handled = await Analytics.handleQueue( store, { messages: [message], queue: Analytics.queueName, }, { onResult() { throw new Error('metrics failed') }, }, ) expect(handled).toBe(true) expect(message.ack).toHaveBeenCalledTimes(1) expect(message.retry).not.toHaveBeenCalled() expect(console.error).toHaveBeenCalledWith( 'Analytics queue result callback failed', expect.objectContaining({ message: 'metrics failed' }), ) }) test('resolves factory sources at the leaf', async () => { const { insert, message, store } = fixture() await Analytics.handleQueue(() => store, { messages: [message], queue: Analytics.queueName }) expect(insert).toHaveBeenCalledTimes(1) }) }) describe('readProjectUsage', () => { test('scopes attributed project rows without current key inventory', async () => { const fetch = vi .fn() .mockImplementation(async () => new Response(JSON.stringify({ data: [] }), { status: 200 })) vi.stubGlobal('fetch', fetch) const usage = await RequestEvents.readProjectUsage(Analytics.clickhouse(clickHouse), { attributions: [{ environment: 'sandbox', projectId: 'prj_1' }], from: '2026-01-01T00:00:00.000Z', interval: 'day', orgId: 'org_1', to: '2026-01-02T00:00:00.000Z', }) const query = String(fetch.mock.calls[0]![1]?.body) expect(query).toContain(`org_id = 'org_1'`) expect(query).toContain(`project_id = 'prj_1'`) expect(query).toContain(`key_environment = 'sandbox'`) expect(query).not.toContain('key_id IN') expect(query).toContain('LIMIT 100') expect(query).toContain('status >= 400 OR rpc_error_count > 0') expect(new URL(String(fetch.mock.calls[0]![0])).searchParams.get('query')).toBeNull() expect(usage).toMatchObject({ from: '2026-01-01T00:00:00.000Z', interval: 'day', to: '2026-01-02T00:00:00.000Z', totals: { averageDurationMs: 0, errors: 0, requests: 0 }, }) }) test('includes JSON-RPC failures in the error breakdown', async () => { const fetch = vi.fn().mockResolvedValue( new Response(JSON.stringify({ data: [{ code: 'billing_required', requests: '2' }] }), { status: 200, }), ) vi.stubGlobal('fetch', fetch) const breakdown = await RequestEvents.readErrorBreakdown(Analytics.clickhouse(clickHouse), { attributions: [{ projectId: 'prj_1' }], from: '2026-01-01T00:00:00.000Z', interval: 'day', orgId: 'org_1', to: '2026-01-02T00:00:00.000Z', }) expect(fetch.mock.calls[0]![1]?.body).toEqual(expect.stringContaining('rpc_error_data_code')) expect(fetch.mock.calls[0]![1]?.body).toEqual( expect.stringContaining('status >= 400 OR rpc_error_count > 0'), ) expect(breakdown).toEqual([{ code: 'billing_required', requests: 2 }]) }) test('keeps legacy rows scoped to project keys without global org filtering', async () => { const fetch = vi .fn() .mockImplementation(async () => new Response(JSON.stringify({ data: [] }), { status: 200 })) vi.stubGlobal('fetch', fetch) await RequestEvents.readProjectUsage(Analytics.clickhouse(clickHouse), { attributions: [ { environment: 'sandbox', projectId: 'prj_1' }, { apiKeyIds: ['key_1', 'key_2'], environment: 'sandbox', projectId: null }, ], from: '2026-01-01T00:00:00.000Z', interval: 'hour', orgId: 'org_1', to: '2026-01-01T01:00:00.000Z', }) const query = String(fetch.mock.calls[0]![1]?.body) expect(query).toContain(`(org_id = 'org_1' AND project_id = 'prj_1'`) expect(query).toContain(`(project_id IS NULL AND key_id IN ('key_1', 'key_2')`) expect(query).toContain(`(key_environment = 'sandbox' OR key_environment IS NULL)`) expect(query).not.toContain(`org_id = 'org_1' AND project_id IS NULL`) }) test('scopes to the whole organization when a project id is absent', async () => { const fetch = vi .fn() .mockImplementation(async () => new Response(JSON.stringify({ data: [] }), { status: 200 })) vi.stubGlobal('fetch', fetch) await RequestEvents.readProjectUsage(Analytics.clickhouse(clickHouse), { attributions: [ { environment: 'sandbox' }, { apiKeyIds: ['key_1'], environment: 'sandbox', projectId: null }, ], from: '2026-01-01T00:00:00.000Z', interval: 'day', orgId: 'org_1', to: '2026-01-02T00:00:00.000Z', }) const query = String(fetch.mock.calls[0]![1]?.body) // Org-wide term matches any project without a `project_id` predicate. expect(query).toContain(`(org_id = 'org_1' AND key_environment = 'sandbox')`) expect(query).toContain(`(project_id IS NULL AND key_id IN ('key_1')`) expect(query).not.toContain(`project_id = 'prj`) }) test.each(['production', 'sandbox'] as const)( 'readBillableCounts dedupes on request_id and gates %s correctly', async (environment) => { const fetch = vi .fn() .mockResolvedValue(new Response(JSON.stringify({ data: [] }), { status: 200 })) vi.stubGlobal('fetch', fetch) await RequestEvents.readBillableCounts(Analytics.clickhouse(clickHouse), { environment, from: '2026-01-01T00:00:00.000Z', to: '2026-01-01T12:00:00.000Z', }) const query = String(fetch.mock.calls[0]![1]?.body) // Deduped, never a raw count(), so queue retries never over-bill. expect(query).toContain('uniqExact(request_id)') expect(query).toContain(`principal_type = 'api_key'`) expect(query).toContain(`key_environment = '${environment}'`) // Sandbox rows bill only when billing was active at request time. if (environment === 'sandbox') expect(query).toContain('billing_active = 1') else expect(query).not.toContain('billing_active = 1') }, ) test('maps grouped rows into the usage response', async () => { const fetch = vi.fn().mockImplementation(async (_url, init) => { const sql = String(init?.body) const data = (() => { if (sql.includes('GROUP BY key_id')) return [{ averageDurationMs: '12.3', errors: '1', key_id: 'key_1', requests: '10' }] if (sql.includes('GROUP BY route')) return [{ averageDurationMs: 12.3, errors: 1, requests: 10, route: '/v1/tokens' }] if (sql.includes('GROUP BY status')) return [ { requests: 9, status: 200 }, { requests: '1', status: '500' }, ] if (sql.includes('formatDateTime')) return [{ errors: 1, requests: 10, time: '2026-01-01T00:00:00.000Z' }] return [{ averageDurationMs: '12.3', errors: '1', requests: '10' }] })() return new Response(JSON.stringify({ data }), { status: 200 }) }) vi.stubGlobal('fetch', fetch) const usage = await RequestEvents.readProjectUsage(Analytics.clickhouse(clickHouse), { attributions: [{ projectId: 'prj_1' }], from: '2026-01-01T00:00:00.000Z', interval: 'day', orgId: 'org_1', to: '2026-01-02T00:00:00.000Z', }) expect(usage).toEqual({ byKey: [{ apiKeyId: 'key_1', averageDurationMs: 12.3, errors: 1, requests: 10 }], byRoute: [{ averageDurationMs: 12.3, errors: 1, requests: 10, route: '/v1/tokens' }], byStatus: [ { requests: 9, status: 200 }, { requests: 1, status: 500 }, ], from: '2026-01-01T00:00:00.000Z', interval: 'day', series: [{ errors: 1, requests: 10, time: '2026-01-01T00:00:00.000Z' }], to: '2026-01-02T00:00:00.000Z', totals: { averageDurationMs: 12.3, errors: 1, requests: 10 }, }) }) }) describe('readLastUsedAt', () => { test('returns the latest request timestamp for each requested key', async () => { const fetch = vi.fn().mockImplementation(async (_url, init) => { const sql = init?.body if (typeof sql !== 'string') throw new Error('Expected a SQL request body.') expect(sql).toContain("key_id IN ('key_1', 'key_2')") expect(sql).toContain('max(timestamp)') return new Response( JSON.stringify({ data: [{ key_id: 'key_1', lastUsedAt: '2026-07-21T12:34:56.000Z' }], }), { status: 200 }, ) }) vi.stubGlobal('fetch', fetch) await expect( RequestEvents.readLastUsedAt(Analytics.clickhouse(clickHouse), ['key_1', 'key_2']), ).resolves.toEqual([{ apiKeyId: 'key_1', lastUsedAt: '2026-07-21T12:34:56.000Z' }]) }) test('skips the analytics query when no keys are requested', async () => { const fetch = vi.fn() vi.stubGlobal('fetch', fetch) await expect( RequestEvents.readLastUsedAt(Analytics.clickhouse(clickHouse), []), ).resolves.toEqual([]) expect(fetch).not.toHaveBeenCalled() }) })