/** * Context Window Compaction Benchmark * * Measures compaction cost with a mock provider: * - compaction latency under threshold pressure * - no-op fast path for below-threshold histories * - token reduction below the low-watermark target budget * - single-pass summarization (exactly 1 call) */ import { describe, expect, mock, test } from "bun:test"; // The compactor reads the conversation's image attachments from the DB to // build its manifest; with no images these return empty. mock.module("../persistence/conversation-crud.js", () => ({ setConversationProcessingStartedAt: () => {}, isConversationProcessing: () => false, getMessages: () => [], })); mock.module("../persistence/attachments-store.js", () => ({ getAttachmentMetadataForMessage: () => [], getAttachmentContent: () => null, })); mock.module("../persistence/llm-request-log-store.js", () => ({ recordRequestLog: () => {}, })); import { DEFAULT_CONFIG } from "../config/defaults.js"; import { resolveCallSiteConfig } from "../config/llm-resolver.js"; import { estimatePromptTokens } from "../context/token-estimator.js"; import { ContextWindowManager } from "../plugins/defaults/compaction/window-manager.js"; import type { Message, Provider } from "../providers/types.js"; // Synthetic per-turn timestamp so the model's tail_start resolves to a message // index. Turn 10 anchors a fat verbatim tail near the front, forcing the // deterministic forward-cut to advance to meet the low-watermark budget. function turnTimestamp(turn: number): string { const hour = String(10 + Math.floor(turn / 60)).padStart(2, "0"); const minute = String(turn % 60).padStart(2, "0"); return `2026-05-21 (Thursday) ${hour}:${minute}:00 -05:00 (America/Chicago)`; } function compactionResponse(callIndex: number): string { return ` Compacted summary, pass ${callIndex}. Goals preserved, constraints noted. - Nothing critical pending. `; } function makeSummaryProvider(counter: { calls: number }): Provider { return { name: "mock", async sendMessage() { counter.calls += 1; return { content: [{ type: "text", text: compactionResponse(counter.calls) }], model: "mock-model", usage: { inputTokens: 420, outputTokens: 85 }, stopReason: "end_turn", }; }, }; } function makeLongMessages(turns: number): Message[] { const rows: Message[] = []; for (let i = 0; i < turns; i++) { rows.push({ role: "user", content: [ { type: "text", text: `\ncurrent_time: ${turnTimestamp( i, )}\n\n[U${i}] User message with enough content to estimate tokens. Topic ${ i % 9 }.`, }, ], }); rows.push({ role: "assistant", content: [ { type: "text", text: `[A${i}] Assistant response with relevant content. Result ${ i % 7 }.`, }, ], }); } return rows; } function makeConfig() { return { ...resolveCallSiteConfig("mainAgent", DEFAULT_CONFIG.llm).contextWindow, maxInputTokens: 6000, targetBudgetRatio: 0.4, compactThreshold: 0.6, summaryBudgetRatio: 0.05, }; } function makeManager(provider: Provider): ContextWindowManager { return new ContextWindowManager({ provider, config: makeConfig(), conversationId: "conv-benchmark", }); } describe("Compaction benchmark", () => { test("compaction with mock provider completes under 500ms", async () => { const counter = { calls: 0 }; const provider = makeSummaryProvider(counter); const config = makeConfig(); const manager = makeManager(provider); // 90 turns = 180 messages, well above 60% of 6000 = 3600 threshold const messages = makeLongMessages(90); const before = estimatePromptTokens(messages, "system prompt", { providerName: "mock", }); // Real loader's default compaction.autoThreshold is 0.7. expect(before).toBeGreaterThan(config.maxInputTokens * 0.7); const start = performance.now(); const result = await manager.maybeCompact(messages); const elapsed = performance.now() - start; expect(result.compacted).toBe(true); expect(elapsed).toBeLessThan(500); }); test("below-threshold check returns in under 50ms (no-op)", async () => { const counter = { calls: 0 }; const provider = makeSummaryProvider(counter); const manager = makeManager(provider); // 3 turns = 6 messages, well below threshold const messages = makeLongMessages(3); const start = performance.now(); const result = await manager.maybeCompact(messages); const elapsed = performance.now() - start; expect(result.compacted).toBe(false); expect(result.reason).toBe("below auto threshold"); expect(elapsed).toBeLessThan(50); expect(counter.calls).toBe(0); }); test("compaction reduces tokens below target budget", async () => { const counter = { calls: 0 }; const provider = makeSummaryProvider(counter); const config = makeConfig(); const manager = makeManager(provider); const messages = makeLongMessages(90); const result = await manager.maybeCompact(messages); expect(result.compacted).toBe(true); expect(result.estimatedInputTokens).toBeLessThan( result.previousEstimatedInputTokens, ); // The deterministic forward-cut drives the rebuilt history at or below the // low-watermark budget: maxInputTokens * (targetBudgetRatio - // summaryBudgetRatio). const targetTokens = Math.floor( config.maxInputTokens * (config.targetBudgetRatio - config.summaryBudgetRatio), ); expect(result.estimatedInputTokens).toBeLessThanOrEqual(targetTokens); }); test("single-pass summarization makes exactly 1 summary call", async () => { const counter = { calls: 0 }; const provider = makeSummaryProvider(counter); const manager = makeManager(provider); const messages = makeLongMessages(90); const result = await manager.maybeCompact(messages); expect(result.compacted).toBe(true); expect(result.summaryCalls).toBe(1); expect(result.summaryCalls).toBe(counter.calls); }); });