import { compressToolResultText, truncateOversizedText } from "./truncate"; import type { CompressionStats, CompactionConfig } from "./types"; interface SemanticSourceItem { message: any; entryId?: string; } interface SemanticUnit { items: SemanticSourceItem[]; chars: number; firstEntryId?: string; } function emptyStats(inputMessages: number): CompressionStats { return { inputMessages, outputMessages: 0, recoveredFromBranchEntries: false, groupedTurns: 0, semanticUnits: 0, chunkCount: 0, droppedThinkingBlocks: 0, compressedToolResults: 0, truncatedMessages: 0, truncatedTextBlocks: 0, snippedMessages: 0, snippedSemanticUnits: 0, microcompactedToolResults: 0, collapsedMessages: 0, collapsedSemanticUnits: 0, }; } function roleOf(message: any): string { return String(message?.role ?? ""); } function hasToolCall(message: any): boolean { return Array.isArray(message?.content) && message.content.some((block: any) => block?.type === "toolCall"); } function startsTurn(message: any): boolean { return roleOf(message) === "user"; } export function estimateMessageChars(message: any): number { if (typeof message?.content === "string") return message.content.length; if (!Array.isArray(message?.content)) return 0; let total = 0; for (const block of message.content) { if (block?.type === "text") total += block.text?.length ?? 0; else if (block?.type === "thinking") total += block.thinking?.length ?? 0; else if (block?.type === "toolCall") total += JSON.stringify(block.arguments ?? {}).length; } return total; } function shouldStartNewUnit( current: SemanticUnit | null, next: SemanticSourceItem, options: { groupByTurns: boolean; groupToolCycles: boolean }, ): boolean { if (!current || current.items.length === 0) return false; const nextRole = roleOf(next.message); if (nextRole === "user") return true; if (!options.groupToolCycles) return false; const previous = current.items[current.items.length - 1]; const previousRole = roleOf(previous.message); const previousHasToolCall = hasToolCall(previous.message); const nextHasToolCall = hasToolCall(next.message); const currentHasUser = current.items.some((item) => roleOf(item.message) === "user"); if (nextRole === "assistant" && nextHasToolCall) { if (previousRole === "toolResult") return true; if (previousRole === "assistant" && previousHasToolCall) return true; if (previousRole === "assistant" && !currentHasUser) return true; } if (options.groupByTurns) { return false; } if (nextRole === "assistant" && previousRole === "assistant" && !previousHasToolCall && !nextHasToolCall) { return true; } return false; } function buildSemanticUnits( items: SemanticSourceItem[], options: { groupByTurns: boolean; groupToolCycles: boolean }, ): SemanticUnit[] { const units: SemanticUnit[] = []; let current: SemanticUnit | null = null; const pushCurrent = (): void => { if (!current || current.items.length === 0) return; units.push(current); current = null; }; for (const item of items) { if (shouldStartNewUnit(current, item, options)) { pushCurrent(); } if (!current) { current = { items: [], chars: 0, firstEntryId: item.entryId, }; } current.items.push(item); current.chars += estimateMessageChars(item.message); if (!current.firstEntryId && item.entryId) current.firstEntryId = item.entryId; } pushCurrent(); return units; } function messagesFromUnits(units: SemanticUnit[]): any[] { return units.flatMap((unit) => unit.items.map((item) => item.message)); } function totalChars(messages: any[]): number { return messages.reduce((sum, message) => sum + estimateMessageChars(message), 0); } function normalizeWhitespace(text: string): string { return text.replace(/\s+/g, " ").trim(); } function firstNonEmptyLine(text: string): string { return text .split(/\r?\n/) .map((line) => line.trim()) .find(Boolean) ?? ""; } function limitText(text: string, maxChars: number): string { if (text.length <= maxChars) return text; if (maxChars <= 24) return text.slice(0, maxChars); return `${text.slice(0, maxChars - 21)} … [truncated]`; } function toSnippet(text: string, maxChars = 160): string { return limitText(normalizeWhitespace(firstNonEmptyLine(text) || text), maxChars); } function textFragmentsFromMessage(message: any): string[] { if (typeof message?.content === "string") return [message.content]; if (!Array.isArray(message?.content)) return []; const fragments: string[] = []; for (const block of message.content) { if (block?.type === "text" && typeof block.text === "string") fragments.push(block.text); if (block?.type === "thinking" && typeof block.thinking === "string") fragments.push(block.thinking); if (block?.type === "toolCall") { const args = block.arguments ? JSON.stringify(block.arguments) : ""; fragments.push(`${block.toolName ?? block.name ?? "toolCall"} ${args}`.trim()); } } return fragments; } function pathHintsFromText(text: string): string[] { const matches = text.match(/(?:@)?(?:\.?\.?\/)?[A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_@.-]+)*\.[A-Za-z0-9_-]+/g) ?? []; const filtered = matches .map((value) => value.replace(/^@/, "")) .filter((value) => !/^https?:\/\//i.test(value)) .filter((value) => value.length <= 160); return [...new Set(filtered)]; } function collectPathHints(message: any): string[] { const values = new Set(); for (const fragment of textFragmentsFromMessage(message)) { for (const hint of pathHintsFromText(fragment)) values.add(hint); } if (Array.isArray(message?.content)) { for (const block of message.content) { if (block?.type !== "toolCall") continue; const args = block.arguments ?? {}; for (const key of ["path", "file", "filepath", "target", "from", "to"]) { const value = args?.[key]; if (typeof value === "string") { for (const hint of pathHintsFromText(value)) values.add(hint); } } } } return [...values]; } function collectToolHints(message: any): string[] { const values = new Set(); if (typeof message?.toolName === "string" && message.toolName.trim()) values.add(message.toolName.trim()); if (Array.isArray(message?.content)) { for (const block of message.content) { if (block?.type === "toolCall") { const name = block.toolName ?? block.name; if (typeof name === "string" && name.trim()) values.add(name.trim()); } } } return [...values]; } function pushUniqueLimited(target: string[], values: string[], maxItems: number): void { for (const value of values) { if (!value || target.includes(value)) continue; target.push(value); if (target.length >= maxItems) return; } } function createSyntheticAssistantMessage(text: string): any { return { role: "assistant", content: text, timestamp: Date.now(), }; } function summarizeUnitsLocally( units: SemanticUnit[], options: { title: string; maxChars: number; maxListItems: number; note: string; }, ): string { const userSignals: string[] = []; const assistantSignals: string[] = []; const paths: string[] = []; const tools: string[] = []; let messageCount = 0; for (const unit of units) { for (const item of unit.items) { messageCount++; const role = roleOf(item.message); const snippets = textFragmentsFromMessage(item.message) .map((fragment) => toSnippet(fragment)) .filter(Boolean); if (role === "user") pushUniqueLimited(userSignals, snippets, options.maxListItems); else pushUniqueLimited(assistantSignals, snippets, options.maxListItems); pushUniqueLimited(paths, collectPathHints(item.message), options.maxListItems + 2); pushUniqueLimited(tools, collectToolHints(item.message), options.maxListItems); } } const lines = [ `[${options.title}]`, `Compressed ${units.length} semantic units / ${messageCount} messages locally before model summarization.`, ]; if (userSignals.length > 0) { lines.push("User intents / requests:"); for (const value of userSignals) lines.push(`- ${value}`); } if (assistantSignals.length > 0) { lines.push("Assistant progress / decisions:"); for (const value of assistantSignals) lines.push(`- ${value}`); } if (paths.length > 0) { lines.push(`Files / paths: ${paths.join(", ")}`); } if (tools.length > 0) { lines.push(`Tools involved: ${tools.join(", ")}`); } lines.push(options.note); return limitText(lines.join("\n"), options.maxChars); } function replaceContentWithText(message: any, text: string): any { if (typeof message?.content === "string") { return { ...message, content: text }; } if (!Array.isArray(message?.content)) { return { ...message, content: text }; } return { ...message, content: [{ type: "text", text }], }; } function createMicrocompactPlaceholder(message: any, maxChars: number): string { const paths = collectPathHints(message).slice(0, 4); const tools = collectToolHints(message).slice(0, 4); const fragments = textFragmentsFromMessage(message) .map((fragment) => toSnippet(fragment, 140)) .filter(Boolean) .slice(0, 2); const lines = ["[Older tool result content cleared by microcompact]"]; if (tools.length > 0) lines.push(`tool=${tools.join(", ")}`); if (paths.length > 0) lines.push(`paths=${paths.join(", ")}`); if (fragments.length > 0) lines.push(`preview=${fragments.join(" | ")}`); lines.push("Keep newer tool results and surrounding assistant messages as the canonical detailed context."); return limitText(lines.join("\n"), maxChars); } function applyHistorySnip(messages: any[], config: CompactionConfig["strategy"], stats: CompressionStats): any[] { if (!config.historySnip.enabled) return messages; if (totalChars(messages) <= config.historySnip.targetChars) return messages; const units = buildSemanticUnits( messages.map((message) => ({ message })), { groupByTurns: config.groupByTurns, groupToolCycles: config.groupToolCycles }, ); const keepHead = Math.max(0, config.historySnip.keepHeadUnits); const keepTail = Math.max(0, config.historySnip.keepTailUnits); const omittedStart = keepHead; const omittedEnd = units.length - keepTail; if (omittedEnd - omittedStart < config.historySnip.minOmittedUnits) return messages; const omittedUnits = units.slice(omittedStart, omittedEnd); if (omittedUnits.length === 0) return messages; const note = createSyntheticAssistantMessage( summarizeUnitsLocally(omittedUnits, { title: "History snip summary", maxChars: config.historySnip.maxPreviewChars, maxListItems: 4, note: "This note preserves coarse signals from an omitted middle section so the final summary can stay focused on the earliest goal framing and the latest concrete progress.", }), ); stats.snippedSemanticUnits += omittedUnits.length; stats.snippedMessages += omittedUnits.reduce((sum, unit) => sum + unit.items.length, 0); return [ ...messagesFromUnits(units.slice(0, omittedStart)), note, ...messagesFromUnits(units.slice(omittedEnd)), ]; } function applyMicrocompact(messages: any[], config: CompactionConfig["strategy"], stats: CompressionStats): any[] { if (!config.microcompact.enabled) return messages; if (totalChars(messages) <= config.microcompact.targetChars) return messages; const candidates = messages .map((message, index) => ({ message, index })) .filter(({ message }) => roleOf(message) === "toolResult" && estimateMessageChars(message) > 0); if (candidates.length <= config.microcompact.keepRecentToolResults) return messages; const keepStart = Math.max(0, candidates.length - config.microcompact.keepRecentToolResults); const replaceable = candidates.slice(0, keepStart); const nextMessages = [...messages]; for (const candidate of replaceable) { if (totalChars(nextMessages) <= config.microcompact.targetChars) break; const previous = nextMessages[candidate.index]; const placeholder = createMicrocompactPlaceholder(previous, config.microcompact.maxPlaceholderChars); const replacement = replaceContentWithText(previous, placeholder); if (estimateMessageChars(replacement) >= estimateMessageChars(previous)) continue; nextMessages[candidate.index] = replacement; stats.microcompactedToolResults++; stats.compressedToolResults++; } return nextMessages; } function applyContextCollapse(messages: any[], config: CompactionConfig["strategy"], stats: CompressionStats): any[] { if (!config.contextCollapse.enabled) return messages; if (totalChars(messages) <= config.contextCollapse.targetChars) return messages; const units = buildSemanticUnits( messages.map((message) => ({ message })), { groupByTurns: config.groupByTurns, groupToolCycles: config.groupToolCycles }, ); const keepFirst = Math.max(0, config.contextCollapse.keepFirstUnits); const keepRecent = Math.max(1, config.contextCollapse.keepRecentUnits); if (units.length <= keepFirst + keepRecent) return messages; const collapseStart = keepFirst; const collapseEnd = Math.max(collapseStart, units.length - keepRecent); const collapsedUnits = units.slice(collapseStart, collapseEnd); if (collapsedUnits.length === 0) return messages; const projection = createSyntheticAssistantMessage( summarizeUnitsLocally(collapsedUnits, { title: "Context collapse projection", maxChars: config.contextCollapse.maxSummaryChars, maxListItems: config.contextCollapse.maxListItems, note: "This is a local projection of older conversation state. Treat it as compressed evidence about earlier work, not as a fresh assistant response.", }), ); stats.collapsedSemanticUnits += collapsedUnits.length; stats.collapsedMessages += collapsedUnits.reduce((sum, unit) => sum + unit.items.length, 0); return [ ...messagesFromUnits(units.slice(0, collapseStart)), projection, ...messagesFromUnits(units.slice(collapseEnd)), ]; } export function extractMessagesFromEntries( entries: any[], options: { keptBudgetChars: number; groupByTurns: boolean; groupToolCycles: boolean }, ): { messages: any[]; firstKeptEntryId: string; groupedTurns: number; semanticUnits: number } { const messageEntries: SemanticSourceItem[] = []; for (const entry of entries) { if (entry?.type === "message" && entry.message) { messageEntries.push({ entryId: entry.id, message: entry.message }); } } const groupedTurns = messageEntries.filter((item) => startsTurn(item.message)).length || (messageEntries.length > 0 ? 1 : 0); const units = buildSemanticUnits(messageEntries, options); if (units.length <= 1) { return { messages: [], firstKeptEntryId: units[0]?.firstEntryId ?? messageEntries[0]?.entryId ?? entries[0]?.id ?? "", groupedTurns, semanticUnits: units.length, }; } let budget = options.keptBudgetChars; let cutoffUnitIndex = units.length; for (let i = units.length - 1; i >= 0; i--) { if (budget - units[i].chars < 0) { cutoffUnitIndex = i + 1; break; } budget -= units[i].chars; if (i === 0) cutoffUnitIndex = 0; } if (cutoffUnitIndex <= 0) { return { messages: [], firstKeptEntryId: units[0]?.firstEntryId ?? messageEntries[0]?.entryId ?? entries[0]?.id ?? "", groupedTurns, semanticUnits: units.length, }; } if (cutoffUnitIndex >= units.length) { cutoffUnitIndex = units.length - 1; } return { messages: units.slice(0, cutoffUnitIndex).flatMap((unit) => unit.items.map((item) => item.message)), firstKeptEntryId: units[cutoffUnitIndex]?.firstEntryId ?? messageEntries[0]?.entryId ?? entries[0]?.id ?? "", groupedTurns, semanticUnits: units.length, }; } export function normalizeMessagesForSummary( messages: any[], config: CompactionConfig["strategy"], ): { messages: any[]; stats: CompressionStats } { const stats = emptyStats(messages.length); const normalized: any[] = []; for (const message of messages) { const role = roleOf(message); if (typeof message?.content === "string") { const normalizedText = role === "toolResult" ? compressToolResultText(message.content, config.toolResults) : truncateOversizedText(message.content, config.oversizedMessages); if (normalizedText.truncated) { if (role === "toolResult") stats.compressedToolResults++; else stats.truncatedMessages++; } normalized.push({ ...message, content: role === "toolResult" ? [{ type: "text", text: normalizedText.text }] : normalizedText.text, }); continue; } if (!Array.isArray(message?.content)) { normalized.push(message); continue; } const blocks: any[] = []; let compressedToolResult = false; let truncatedTextBlock = false; for (const block of message.content) { if (block?.type === "thinking" && config.dropRawThinking) { stats.droppedThinkingBlocks++; continue; } if (block?.type === "text") { const value = String(block.text ?? ""); const normalizedText = role === "toolResult" ? compressToolResultText(value, config.toolResults) : truncateOversizedText(value, config.oversizedMessages); if (normalizedText.truncated) { if (role === "toolResult") compressedToolResult = true; else truncatedTextBlock = true; } blocks.push({ ...block, text: normalizedText.text }); continue; } blocks.push(block); } if (compressedToolResult) stats.compressedToolResults++; if (truncatedTextBlock) stats.truncatedTextBlocks++; if (blocks.length === 0) continue; normalized.push({ ...message, content: blocks }); } let prepared = normalized; prepared = applyHistorySnip(prepared, config, stats); prepared = applyMicrocompact(prepared, config, stats); prepared = applyContextCollapse(prepared, config, stats); stats.outputMessages = prepared.length; stats.semanticUnits = buildSemanticUnits( prepared.map((message) => ({ message })), { groupByTurns: config.groupByTurns, groupToolCycles: config.groupToolCycles }, ).length; return { messages: prepared, stats }; } export function chunkMessagesForSummary( messages: any[], config: CompactionConfig["strategy"], stats?: CompressionStats, ): any[][] { const units = buildSemanticUnits( messages.map((message) => ({ message })), { groupByTurns: config.groupByTurns, groupToolCycles: config.groupToolCycles }, ); if (stats) { stats.semanticUnits = units.length; } if (!config.chunking.enabled || units.length <= 1) { if (stats) stats.chunkCount = messages.length > 0 ? 1 : 0; return messages.length > 0 ? [messages] : []; } const totalUnitChars = units.reduce((sum, unit) => sum + unit.chars, 0); if (totalUnitChars <= config.chunking.targetChars) { if (stats) stats.chunkCount = messages.length > 0 ? 1 : 0; return messages.length > 0 ? [messages] : []; } const chunks: SemanticUnit[][] = []; let currentChunk: SemanticUnit[] = []; let currentChars = 0; let currentMessages = 0; const pushChunk = (): void => { if (currentChunk.length === 0) return; chunks.push(currentChunk); currentChunk = []; currentChars = 0; currentMessages = 0; }; for (const unit of units) { const shouldSplit = currentChunk.length > 0 && currentChars + unit.chars > config.chunking.targetChars && currentMessages >= config.chunking.minMessagesPerChunk; if (shouldSplit) { pushChunk(); } currentChunk.push(unit); currentChars += unit.chars; currentMessages += unit.items.length; } pushChunk(); while (chunks.length > config.chunking.maxChunks) { const tail = chunks.pop(); if (!tail || chunks.length === 0) break; chunks[chunks.length - 1].push(...tail); } const result = chunks.map((chunk) => chunk.flatMap((unit) => unit.items.map((item) => item.message))); if (stats) stats.chunkCount = result.length; return result; }