import http from 'node:http'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { performance } from 'node:perf_hooks'; import { pathToFileURL } from 'node:url'; type SampleSummary = { runs: number; samplesMs: number[]; minMs: number; p50Ms: number; p95Ms: number; maxMs: number; avgMs: number; }; type ThreadHistoryPayload = { messages?: unknown[]; events?: unknown[]; }; type HttpHistorySample = { totalMs: number; status: number; bytes: number; jsonParseMs: number; replayMs: number; messages: number; events: number; uiMessages: number; }; type SseSample = { responseMs: number; firstEventMs: number; drainMs: number; status: number; events: number; bytes: number; }; type BenchMessageHistoryOptions = { baseUrl?: string; threadId?: string; computerId: string; apiKey?: string; authorization?: string; runs: number; limit: number; streamSampleMs: number; }; export function summarize(samples: number[]): SampleSummary { const sorted = [...samples].sort((a, b) => a - b); const percentile = (value: number) => { if (sorted.length === 0) return 0; return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * value) - 1)] ?? 0; }; const avg = samples.reduce((sum, sample) => sum + sample, 0) / Math.max(1, samples.length); return { runs: samples.length, samplesMs: samples.map((sample) => Number(sample.toFixed(3))), minMs: Number((sorted[0] ?? 0).toFixed(3)), p50Ms: Number(percentile(0.5).toFixed(3)), p95Ms: Number(percentile(0.95).toFixed(3)), maxMs: Number((sorted.at(-1) ?? 0).toFixed(3)), avgMs: Number(avg.toFixed(3)), }; } function readPositiveInteger(value: string | undefined, fallback: number): number { const parsed = Number.parseInt(value || '', 10); return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } export function parseBenchMessageHistoryArgs(argv: string[]): BenchMessageHistoryOptions { const options: BenchMessageHistoryOptions = { baseUrl: process.env.BENCH_MESSAGE_HISTORY_REMOTE_BASE_URL?.trim(), threadId: process.env.BENCH_MESSAGE_HISTORY_REMOTE_THREAD_ID?.trim(), computerId: process.env.BENCH_MESSAGE_HISTORY_REMOTE_COMPUTER_ID?.trim() || 'local', apiKey: process.env.BENCH_MESSAGE_HISTORY_REMOTE_API_KEY?.trim(), authorization: ( process.env.BENCH_MESSAGE_HISTORY_REMOTE_AUTHORIZATION || (process.env.BENCH_MESSAGE_HISTORY_REMOTE_BEARER ? `Bearer ${process.env.BENCH_MESSAGE_HISTORY_REMOTE_BEARER}` : '') ).trim() || undefined, runs: readPositiveInteger(process.env.BENCH_MESSAGE_HISTORY_RUNS, 12), limit: readPositiveInteger(process.env.BENCH_MESSAGE_HISTORY_LIMIT, 400), streamSampleMs: readPositiveInteger(process.env.BENCH_MESSAGE_HISTORY_STREAM_SAMPLE_MS, 5_000), }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; const next = argv[index + 1]; if (arg === '--base-url') { options.baseUrl = next?.trim(); index += 1; } else if (arg === '--thread-id') { options.threadId = next?.trim(); index += 1; } else if (arg === '--computer-id') { options.computerId = next?.trim() || options.computerId; index += 1; } else if (arg === '--api-key') { options.apiKey = next?.trim(); index += 1; } else if (arg === '--authorization') { options.authorization = next?.trim(); index += 1; } else if (arg === '--runs') { options.runs = readPositiveInteger(next, options.runs); index += 1; } else if (arg === '--limit') { options.limit = readPositiveInteger(next, options.limit); index += 1; } else if (arg === '--stream-sample-ms') { options.streamSampleMs = readPositiveInteger(next, options.streamSampleMs); index += 1; } } return options; } export function buildCodexThreadEndpoint( baseUrl: string, threadId: string, endpoint: 'messages' | 'stream', params: Record = {}, computerId = 'local', ): string { const normalizedBaseUrl = baseUrl.replace(/\/+$/, ''); const url = new URL( `${normalizedBaseUrl}/api/computers/${encodeURIComponent(computerId)}/agents/codex/threads/${encodeURIComponent(threadId)}/${endpoint}`, ); for (const [key, value] of Object.entries(params)) { url.searchParams.set(key, value); } return url.toString(); } function summarizeField(samples: T[], selector: (sample: T) => number): SampleSummary { return summarize(samples.map(selector)); } function denseTimestamp(index: number, suffixMs = 0): string { return `2026-06-04T12:${String(Math.floor(index / 60)).padStart(2, '0')}:${String(index % 60).padStart(2, '0')}.${String(suffixMs).padStart(3, '0')}Z`; } function measure(runs: number, fn: () => T): { summary: SampleSummary; last: T } { const samples: number[] = []; let last: T; for (let run = 0; run < runs; run += 1) { const startedAt = performance.now(); last = fn(); samples.push(performance.now() - startedAt); } return { summary: summarize(samples), last: last! }; } function warmup(runs: number, fn: () => void): void { for (let run = 0; run < runs; run += 1) { fn(); } } async function measureAsync(runs: number, fn: () => Promise): Promise<{ samples: T[]; total: SampleSummary }> { const samples: T[] = []; const totals: number[] = []; for (let run = 0; run < runs; run += 1) { const startedAt = performance.now(); samples.push(await fn()); totals.push(performance.now() - startedAt); } return { samples, total: summarize(totals) }; } async function warmupAsync(runs: number, fn: () => Promise): Promise { for (let run = 0; run < runs; run += 1) { await fn(); } } function createFixture(agentHome: string, supenHome: string) { const threadId = '019e8b00-0000-7000-9000-000000000099'; const threadsDir = path.join(agentHome, 'threads', '2026', '06', '04'); fs.mkdirSync(threadsDir, { recursive: true }); fs.writeFileSync( path.join(agentHome, 'thread_index.jsonl'), `${JSON.stringify({ id: threadId, thread_name: 'Benchmark thread', updated_at: '2026-06-04T12:00:00.000Z' })}\n`, 'utf-8', ); const threadPath = path.join(threadsDir, `rollout-2026-06-04T12-00-00-${threadId}.jsonl`); const largeAssistantText = 'assistant-detail '.repeat(1024); const lines: string[] = []; for (let index = 0; index < 250; index += 1) { lines.push(JSON.stringify({ timestamp: denseTimestamp(index, 0), type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: `request ${index}` }], }, })); lines.push(JSON.stringify({ timestamp: denseTimestamp(index, 100), type: 'response_item', payload: { type: 'function_call', name: 'exec_command', call_id: `call-${index}`, arguments: JSON.stringify({ cmd: `echo ${index}` }), }, })); lines.push(JSON.stringify({ timestamp: denseTimestamp(index, 200), type: 'response_item', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: `answer ${index}\n${largeAssistantText}` }], }, })); } fs.writeFileSync(threadPath, `${lines.join('\n')}\n`, 'utf-8'); const eventLogDir = path.join(supenHome, 'threads', threadId, 'event-log'); fs.mkdirSync(eventLogDir, { recursive: true }); const rawEvents: string[] = []; for (let index = 0; index < 120; index += 1) { rawEvents.push(JSON.stringify({ event_id: `bench-event-${index}`, runtime_event_id: `bench-runtime-event-${index}`, thread_id: threadId, runtime_thread_id: threadId, sequence: index + 1, runtime_sequence: index + 1, source: 'codex-app-server', event_type: 'item/agentMessage/delta', raw_payload: { type: 'data-codex-event', data: { eventType: 'item/agentMessage/delta', raw: { method: 'item/agentMessage/delta', params: { threadId, turnId: 'turn-bench', itemId: 'assistant-bench', delta: `stream-delta-${index} `, }, }, }, }, received_at: denseTimestamp(index, 300), payload_hash: `bench-${index}`, })); } fs.writeFileSync(path.join(eventLogDir, 'raw-events.jsonl'), `${rawEvents.join('\n')}\n`, 'utf-8'); fs.writeFileSync( path.join(eventLogDir, 'head.json'), `${JSON.stringify({ last_sequence: rawEvents.length, updated_at: denseTimestamp(rawEvents.length, 0) })}\n`, 'utf-8', ); return { threadId, threadPath, jsonlLines: lines.length, seededStreamEvents: rawEvents.length, }; } async function startLoopbackServer() { const { dispatchRequest } = await import('../src/http/router.js'); const { setCorsHeaders } = await import('../src/http/context.js'); const server = http.createServer(async (req, res) => { if (req.method === 'OPTIONS') { setCorsHeaders(req, res); res.writeHead(204); res.end(); return; } setCorsHeaders(req, res); await dispatchRequest(req, res, () => {}); }); await new Promise((resolve, reject) => { server.listen(0, '127.0.0.1', resolve); server.on('error', reject); }); const address = server.address(); if (!address || typeof address === 'string') { throw new Error('Expected loopback TCP server address.'); } return { server, baseUrl: `http://127.0.0.1:${address.port}`, }; } async function measureHttpHistory( url: string, buildThreadUIMessages: (payload: ThreadHistoryPayload, options: { includeSilentEvents: boolean }) => unknown[], headers: HeadersInit = {}, ): Promise { const startedAt = performance.now(); const response = await fetch(url, { headers }); const text = await response.text(); const totalMs = performance.now() - startedAt; if (!response.ok) { throw new Error(`History request failed (${response.status}) for ${url}: ${text.slice(0, 300)}`); } const parseStartedAt = performance.now(); const history = JSON.parse(text) as ThreadHistoryPayload; const jsonParseMs = performance.now() - parseStartedAt; const replayStartedAt = performance.now(); const uiMessages = buildThreadUIMessages(history, { includeSilentEvents: false }); const replayMs = performance.now() - replayStartedAt; return { totalMs, status: response.status, bytes: Buffer.byteLength(text), jsonParseMs, replayMs, messages: history.messages?.length ?? 0, events: history.events?.length ?? 0, uiMessages: uiMessages.length, }; } async function measureSse( url: string, headers: HeadersInit = {}, options: { targetEvents?: number; sampleMs?: number } = {}, ): Promise { const startedAt = performance.now(); const targetEvents = Math.max(1, options.targetEvents ?? 120); const sampleMs = Math.max(1, options.sampleMs ?? 5_000); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), sampleMs); let response: Response; try { response = await fetch(url, { headers, signal: controller.signal }); } catch (error) { clearTimeout(timeout); if ((error as Error).name === 'AbortError') { const elapsedMs = performance.now() - startedAt; return { responseMs: elapsedMs, firstEventMs: 0, drainMs: elapsedMs, status: 0, events: 0, bytes: 0, }; } throw error; } const responseMs = performance.now() - startedAt; if (!response.ok) { const body = await response.text().catch(() => ''); clearTimeout(timeout); throw new Error(`SSE request failed (${response.status}) for ${url}: ${body.slice(0, 300)}`); } if (!response.body) { clearTimeout(timeout); throw new Error(`SSE response body missing (${response.status})`); } const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ''; let bytes = 0; let events = 0; let firstEventMs = 0; const processLine = (line: string) => { if (!line.startsWith('data:')) return; const data = line.slice(5).trim(); if (!data) return; events += 1; if (firstEventMs === 0) firstEventMs = performance.now() - startedAt; }; try { while (events < targetEvents) { const { done, value } = await reader.read(); if (done) break; bytes += value.byteLength; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split(/\r?\n/); buffer = lines.pop() ?? ''; for (const line of lines) processLine(line); } } catch (error) { if ((error as Error).name !== 'AbortError') throw error; } finally { clearTimeout(timeout); } await reader.cancel().catch(() => {}); return { responseMs, firstEventMs, drainMs: performance.now() - startedAt, status: response.status, events, bytes, }; } function sampleShape>(sample: T | undefined): T | undefined { if (!sample) return undefined; return Object.fromEntries( Object.entries(sample).map(([key, value]) => [ key, typeof value === 'number' ? Number(value.toFixed(3)) : value, ]), ) as T; } async function main() { const options = parseBenchMessageHistoryArgs(process.argv.slice(2)); const agentHome = fs.mkdtempSync(path.join(os.tmpdir(), 'supen-message-history-bench-')); const supenHome = path.join(agentHome, '.supen'); process.env.CODEX_HOME = agentHome; process.env.HOME = agentHome; process.env.SUPEN_HOME = supenHome; process.env.HTTP_API_KEY = ''; process.env.NODE_ENV = 'test'; let server: http.Server | null = null; try { const fixture = createFixture(agentHome, supenHome); const { readCodexThreadHistory } = await import('../src/http/routes/system.js'); const { directHttpAccessToken } = await import('../src/core/direct-http-access.js'); const { buildThreadUIMessages } = await import('../../app/app/lib/chat-ui-message-replay.ts'); const requestedHistoryLimit = options.limit; warmup(2, () => { readCodexThreadHistory(fixture.threadId, requestedHistoryLimit); }); const historyBench = measure(options.runs, () => readCodexThreadHistory(fixture.threadId, requestedHistoryLimit)); const history = historyBench.last ?? { messages: [], events: [] }; warmup(5, () => { buildThreadUIMessages(history, { includeSilentEvents: false }); }); const replayBench = measure(80, () => buildThreadUIMessages(history, { includeSilentEvents: false }), ); const loopback = await startLoopbackServer(); server = loopback.server; const directHeaders = { Origin: 'https://hub.supen.ai', 'X-API-Key': directHttpAccessToken(), }; const historyUrl = buildCodexThreadEndpoint(loopback.baseUrl, fixture.threadId, 'messages', { limit: String(requestedHistoryLimit), eventLimit: '200', }); const streamUrl = buildCodexThreadEndpoint(loopback.baseUrl, fixture.threadId, 'stream', { after: '0', }); await warmupAsync(2, async () => { await measureHttpHistory(historyUrl, buildThreadUIMessages, directHeaders); }); const httpHistoryBench = await measureAsync(options.runs, () => measureHttpHistory(historyUrl, buildThreadUIMessages, directHeaders), ); const sseBench = await measureAsync(Math.max(1, Math.min(options.runs, 8)), () => measureSse(streamUrl, directHeaders, { targetEvents: fixture.seededStreamEvents, sampleMs: options.streamSampleMs, }), ); const remoteBaseUrl = (options.baseUrl || '').trim().replace(/\/+$/, ''); const remoteThreadId = (options.threadId || '').trim(); const remoteComputerId = options.computerId.trim() || 'local'; const remoteHeaders: Record = {}; if (options.apiKey) remoteHeaders['X-API-Key'] = options.apiKey; if (options.authorization) remoteHeaders.Authorization = options.authorization; const shouldRunRemote = Boolean(remoteBaseUrl && remoteThreadId); const remoteHistoryUrl = shouldRunRemote ? buildCodexThreadEndpoint( remoteBaseUrl, remoteThreadId, 'messages', { limit: String(requestedHistoryLimit), eventLimit: '200' }, remoteComputerId, ) : ''; const remoteStreamUrl = shouldRunRemote ? buildCodexThreadEndpoint(remoteBaseUrl, remoteThreadId, 'stream', { after: '0' }, remoteComputerId) : ''; const remoteRuns = Math.max(1, Math.min(options.runs, 6)); const remoteHistoryBench = shouldRunRemote ? await measureAsync(remoteRuns, () => measureHttpHistory(remoteHistoryUrl, buildThreadUIMessages, remoteHeaders)) : null; const shouldRunRemoteStream = shouldRunRemote && process.env.BENCH_MESSAGE_HISTORY_REMOTE_STREAM !== '0'; const remoteSseBench = shouldRunRemoteStream ? await measureAsync(Math.max(1, Math.min(options.runs, 3)), () => measureSse(remoteStreamUrl, remoteHeaders, { targetEvents: Number.parseInt(process.env.BENCH_MESSAGE_HISTORY_REMOTE_STREAM_EVENTS || '1', 10) || 1, sampleMs: options.streamSampleMs, }), ) : null; console.log(JSON.stringify({ benchmark: { generatedAt: new Date().toISOString(), options, }, fixture: { threadJsonlBytes: fs.statSync(fixture.threadPath).size, jsonlLines: fixture.jsonlLines, seededStreamEvents: fixture.seededStreamEvents, requestedHistoryLimit, loadedMessages: history.messages.length, loadedEvents: history.events.length, replayMessages: replayBench.last.length, }, inProcess: { daemonHistory: historyBench.summary, frontendReplay: replayBench.summary, }, loopbackHttp: { historyTotal: summarizeField(httpHistoryBench.samples, (sample) => sample.totalMs), historyJsonParse: summarizeField(httpHistoryBench.samples, (sample) => sample.jsonParseMs), historyReplay: summarizeField(httpHistoryBench.samples, (sample) => sample.replayMs), last: sampleShape(httpHistoryBench.samples.at(-1)), }, loopbackSse: { response: summarizeField(sseBench.samples, (sample) => sample.responseMs), firstEvent: summarizeField(sseBench.samples, (sample) => sample.firstEventMs), drain: summarizeField(sseBench.samples, (sample) => sample.drainMs), last: sampleShape(sseBench.samples.at(-1)), }, compareRemote: { ...(remoteHistoryBench ? { baseUrl: remoteBaseUrl, computerId: remoteComputerId, threadId: remoteThreadId, historyTotal: summarizeField(remoteHistoryBench.samples, (sample) => sample.totalMs), historyJsonParse: summarizeField(remoteHistoryBench.samples, (sample) => sample.jsonParseMs), historyReplay: summarizeField(remoteHistoryBench.samples, (sample) => sample.replayMs), lastHistory: sampleShape(remoteHistoryBench.samples.at(-1)), ...(remoteSseBench ? { sseResponse: summarizeField(remoteSseBench.samples, (sample) => sample.responseMs), sseFirstEvent: summarizeField(remoteSseBench.samples, (sample) => sample.firstEventMs), sseDrain: summarizeField(remoteSseBench.samples, (sample) => sample.drainMs), lastSse: sampleShape(remoteSseBench.samples.at(-1)), } : { sseSkipped: 'Remote SSE timing is enabled by default when a remote base URL and thread ID are provided. BENCH_MESSAGE_HISTORY_REMOTE_STREAM=0 skips it.', }), } : { skipped: true, note: 'Set --base-url plus --thread-id or BENCH_MESSAGE_HISTORY_REMOTE_BASE_URL plus BENCH_MESSAGE_HISTORY_REMOTE_THREAD_ID. Optional: --computer-id, --api-key, --authorization, BENCH_MESSAGE_HISTORY_REMOTE_COMPUTER_ID/API_KEY/AUTHORIZATION/BEARER.', }), }, }, null, 2)); } finally { if (server) { await new Promise((resolve) => server!.close(() => resolve())); } fs.rmSync(agentHome, { recursive: true, force: true }); } } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { main().catch((error) => { console.error(error); process.exitCode = 1; }); }