import { createHash } from 'node:crypto'; import type { Message } from '@earendil-works/pi-ai'; import { canonicalJson } from '../attested-pi-run.js'; /** * Frozen `visible-conversation-ledger-v2` transform. * * This is the single shared implementation of the package's conversation * projection: visible user/assistant text verbatim, assistant thinking and all * tool traffic replaced with deterministic hash-accounted omission receipts, * images marker-only, unknown block types a loud error. * * The transform has **no behavioural knobs**. Every consumer receives exactly * the same disposition for exactly the same input. Consumer identity — schema * versions, policy ids, request authority, branch-filter provenance — lives * entirely outside this module, so a consumer can never claim behaviour that did * not occur. A semantic change requires a new transform id and a new * implementation, never a flag added here. * * Fusion's persisted artifact bytes are frozen against this implementation and * are proven bit-identical by `tests/unit/fusion-golden-bytes.test.ts`. */ export const VISIBLE_CONVERSATION_TRANSFORM_ID = 'visible-conversation-ledger-v2'; export const CONVERSATION_IMAGE_OMISSION_PREFIX = '[Image omitted from fusion text transcript: '; export const OMITTED_EVENT_KINDS = [ 'assistant_thinking', 'tool_call', 'tool_result_text', 'tool_result_image', ] as const; export type OmittedEventKind = (typeof OMITTED_EVENT_KINDS)[number]; export interface OmittedEventRecord { index: number; source_ordinal: number; block_ordinal: number; kind: OmittedEventKind; payload_bytes: number; payload_sha256: string; tool_name?: string; tool_call_id?: string; mime_type?: string; } export interface OmittedActivityProjectionMapEntry { canonical_entry_index: number; entry_kind: 'omitted_activity'; ledger_index_first: number; ledger_index_last: number; } export interface LedgerOnlyImageProjectionMapEntry { entry_kind: 'ledger_only_tool_result_image'; ledger_index_first: number; ledger_index_last: number; } export type ContextProjectionMapEntry = | OmittedActivityProjectionMapEntry | LedgerOnlyImageProjectionMapEntry; export interface ProjectionTextEntry { kind: 'text'; source_ordinal: number; block_ordinal: number; role: 'user' | 'assistant'; text: string; } /** * Per-kind counts for one omitted run. Zero-valued kinds are omitted from the * serialized receipt by a fixed policy rule so receipt size does not scale with * the number of tracked kinds; absent means exactly zero. */ export interface OmittedRunCounts { assistant_thinking?: number; tool_calls?: number; tool_result_texts?: number; } export interface ProjectionOmissionEntry { kind: 'omitted_activity'; at: readonly [number, number]; bytes: number; counts: OmittedRunCounts; } export type ProjectionEntry = ProjectionTextEntry | ProjectionOmissionEntry; export interface ToolCallNameCount { name: string; calls: number; } export interface ProjectionAccounting { message_count: number; included_text_entry_count: number; included_user_text_bytes: number; included_assistant_text_bytes: number; included_image_marker_count: number; empty_text_block_count: number; omitted_run_count: number; omitted_event_count: number; omitted_thinking_bytes: number; omitted_tool_call_count: number; omitted_tool_call_argument_bytes: number; omitted_tool_result_text_count: number; omitted_tool_result_text_bytes: number; omitted_tool_result_image_count: number; omitted_tool_result_image_bytes: number; tool_call_names: readonly ToolCallNameCount[]; ledger_entry_count: number; ledger_root_sha256: string; omission_receipt_utf8_bytes: number; } /** * Ledger body produced by the transform, carrying no consumer envelope. * * The ledger root commits only to the ledger rows, never to schema version, * policy id, or transform id, so this body can be sealed by different consumers * without changing the root. That property is pinned by a dedicated test. */ export interface LedgerBodyV2 { entries: readonly OmittedEventRecord[]; projection_map: readonly ContextProjectionMapEntry[]; root_sha256: string; } export interface ProjectedConversationV2 { entries: readonly ProjectionEntry[]; accounting: ProjectionAccounting; ledger: LedgerBodyV2; } /** Raised when the transform meets a conversation block it refuses to silently drop. */ export class UnsupportedConversationBlockError extends Error { readonly label: string; constructor(label: string) { super( `fusion context projection encountered an unsupported conversation block: ${label}`, ); this.name = 'UnsupportedConversationBlockError'; this.label = label; } } function utf8Bytes(value: string): number { return Buffer.byteLength(value, 'utf8'); } function sha256Hex(bytes: Buffer): string { return createHash('sha256').update(bytes).digest('hex'); } function uint64be(value: number): Buffer { const out = Buffer.alloc(8); out.writeBigUInt64BE(BigInt(value)); return out; } /** Length-prefixed framing so concatenated fields cannot collide across boundaries. */ function lengthPrefixed(bytes: Buffer): Buffer { return Buffer.concat([uint64be(bytes.length), bytes]); } function ledgerLeafHash(index: number, record: OmittedEventRecord): Buffer { return createHash('sha256') .update(Buffer.from('pi-fusion-ledger-leaf-v1\0', 'utf8')) .update(uint64be(index)) .update(lengthPrefixed(Buffer.from(canonicalJson(record), 'utf8'))) .digest(); } function ledgerRootHash(leaves: readonly Buffer[]): string { const hash = createHash('sha256') .update(Buffer.from('pi-fusion-ledger-root-v1\0', 'utf8')) .update(uint64be(leaves.length)); for (const leaf of leaves) hash.update(leaf); return hash.digest('hex'); } function unsupportedBlock(label: string): UnsupportedConversationBlockError { return new UnsupportedConversationBlockError(label); } /** * Fixed policy rule: a zero-valued kind is absent from the receipt rather than * serialized as `0`. This keeps receipts compact without losing information — * absent is defined by the policy version to mean exactly zero — and it is * applied unconditionally, never adaptively because an input happens to be large. */ function compactCounts(counts: Required): OmittedRunCounts { const out: OmittedRunCounts = {}; if (counts.assistant_thinking > 0) out.assistant_thinking = counts.assistant_thinking; if (counts.tool_calls > 0) out.tool_calls = counts.tool_calls; if (counts.tool_result_texts > 0) out.tool_result_texts = counts.tool_result_texts; return out; } function imageMarker(mimeType: string): string { return `${CONVERSATION_IMAGE_OMISSION_PREFIX}${mimeType}]`; } /** * Accumulates omitted-event ledger rows and turns maximal contiguous non-image * omission runs into compact, source-ordered receipts inside the projection. */ class ProjectionBuilder { private readonly entries: ProjectionEntry[] = []; private readonly ledger: OmittedEventRecord[] = []; private readonly leaves: Buffer[] = []; private readonly projectionMap: ContextProjectionMapEntry[] = []; private readonly toolCallNames = new Map(); private pendingCounts: Required | undefined; private pendingBytes = 0; private pendingLedgerFirst = 0; private pendingLedgerLast = 0; private pendingSourceFirst = 0; private pendingSourceLast = 0; includedUserTextBytes = 0; includedAssistantTextBytes = 0; includedImageMarkers = 0; emptyTextBlocks = 0; addText(entry: ProjectionTextEntry): void { this.flush(); this.entries.push(entry); if (entry.role === 'user') this.includedUserTextBytes += utf8Bytes(entry.text); else this.includedAssistantTextBytes += utf8Bytes(entry.text); } addOmission( sourceOrdinal: number, blockOrdinal: number, kind: OmittedEventKind, payload: Buffer, extra: { toolName?: string; toolCallId?: string; mimeType?: string } = {}, ): void { const index = this.ledger.length; const record: OmittedEventRecord = { index, source_ordinal: sourceOrdinal, block_ordinal: blockOrdinal, kind, payload_bytes: payload.length, payload_sha256: sha256Hex(payload), }; if (extra.toolName !== undefined) record.tool_name = extra.toolName; if (extra.toolCallId !== undefined) record.tool_call_id = extra.toolCallId; if (extra.mimeType !== undefined) record.mime_type = extra.mimeType; this.ledger.push(record); this.leaves.push(ledgerLeafHash(index, record)); if (extra.toolName !== undefined && kind === 'tool_call') { this.toolCallNames.set(extra.toolName, (this.toolCallNames.get(extra.toolName) ?? 0) + 1); } if (kind === 'tool_result_image') { this.flush(); this.addLedgerOnlyImageMap(index); return; } if (this.pendingCounts === undefined) { this.pendingCounts = { assistant_thinking: 0, tool_calls: 0, tool_result_texts: 0, }; this.pendingBytes = 0; this.pendingLedgerFirst = index; this.pendingSourceFirst = sourceOrdinal; } this.pendingLedgerLast = index; this.pendingSourceLast = sourceOrdinal; this.pendingBytes += payload.length; if (kind === 'assistant_thinking') this.pendingCounts.assistant_thinking += 1; else if (kind === 'tool_call') this.pendingCounts.tool_calls += 1; else this.pendingCounts.tool_result_texts += 1; } private addLedgerOnlyImageMap(index: number): void { const previous = this.projectionMap[this.projectionMap.length - 1]; if ( previous !== undefined && previous.entry_kind === 'ledger_only_tool_result_image' && previous.ledger_index_last + 1 === index ) { previous.ledger_index_last = index; return; } this.projectionMap.push({ entry_kind: 'ledger_only_tool_result_image', ledger_index_first: index, ledger_index_last: index, }); } private flush(): void { const counts = this.pendingCounts; if (counts === undefined) return; const canonicalEntryIndex = this.entries.length; const entry: ProjectionOmissionEntry = { at: [this.pendingSourceFirst, this.pendingSourceLast], bytes: this.pendingBytes, counts: compactCounts(counts), kind: 'omitted_activity', }; this.entries.push(entry); this.projectionMap.push({ canonical_entry_index: canonicalEntryIndex, entry_kind: 'omitted_activity', ledger_index_first: this.pendingLedgerFirst, ledger_index_last: this.pendingLedgerLast, }); this.pendingCounts = undefined; this.pendingBytes = 0; } finish(messageCount: number): ProjectedConversationV2 { this.flush(); const rootSha256 = ledgerRootHash(this.leaves); let omittedRunCount = 0; let includedTextEntries = 0; let receiptBytes = 0; for (const entry of this.entries) { if (entry.kind === 'text') includedTextEntries += 1; else { omittedRunCount += 1; receiptBytes += utf8Bytes(canonicalJson(entry)); } } let thinkingBytes = 0; let toolCallCount = 0; let toolArgumentBytes = 0; let toolResultTextCount = 0; let toolResultTextBytes = 0; let toolResultImageCount = 0; let toolResultImageBytes = 0; for (const row of this.ledger) { if (row.kind === 'assistant_thinking') thinkingBytes += row.payload_bytes; else if (row.kind === 'tool_call') { toolCallCount += 1; toolArgumentBytes += row.payload_bytes; } else if (row.kind === 'tool_result_text') { toolResultTextCount += 1; toolResultTextBytes += row.payload_bytes; } else { toolResultImageCount += 1; toolResultImageBytes += row.payload_bytes; } } const toolCallNames: ToolCallNameCount[] = [...this.toolCallNames.entries()] .map(([name, calls]) => ({ name, calls })) .sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0)); const accounting: ProjectionAccounting = { message_count: messageCount, included_text_entry_count: includedTextEntries, included_user_text_bytes: this.includedUserTextBytes, included_assistant_text_bytes: this.includedAssistantTextBytes, included_image_marker_count: this.includedImageMarkers, empty_text_block_count: this.emptyTextBlocks, omitted_run_count: omittedRunCount, omitted_event_count: this.ledger.length, omitted_thinking_bytes: thinkingBytes, omitted_tool_call_count: toolCallCount, omitted_tool_call_argument_bytes: toolArgumentBytes, omitted_tool_result_text_count: toolResultTextCount, omitted_tool_result_text_bytes: toolResultTextBytes, omitted_tool_result_image_count: toolResultImageCount, omitted_tool_result_image_bytes: toolResultImageBytes, tool_call_names: toolCallNames, ledger_entry_count: this.ledger.length, ledger_root_sha256: rootSha256, omission_receipt_utf8_bytes: receiptBytes, }; return { entries: this.entries, accounting, ledger: { entries: this.ledger, projection_map: this.projectionMap, root_sha256: rootSha256, }, }; } } /** * Projects one user-role message. User text is retained verbatim; images stay * marker-only so the child never receives raw bytes. */ function projectUserMessage( builder: ProjectionBuilder, content: Message['content'], sourceOrdinal: number, ): void { if (typeof content === 'string') { if (content.length === 0) { builder.emptyTextBlocks += 1; return; } builder.addText({ kind: 'text', source_ordinal: sourceOrdinal, block_ordinal: 0, role: 'user', text: content, }); return; } for (const [blockOrdinal, block] of content.entries()) { if (block.type === 'text') { if (block.text.length === 0) { builder.emptyTextBlocks += 1; continue; } builder.addText({ kind: 'text', source_ordinal: sourceOrdinal, block_ordinal: blockOrdinal, role: 'user', text: block.text, }); continue; } if (block.type === 'image') { builder.includedImageMarkers += 1; builder.addText({ kind: 'text', source_ordinal: sourceOrdinal, block_ordinal: blockOrdinal, role: 'user', text: imageMarker(block.mimeType), }); continue; } throw unsupportedBlock(`user block ${String(Reflect.get(block, 'type'))}`); } } function projectAssistantMessage( builder: ProjectionBuilder, content: Extract['content'], sourceOrdinal: number, ): void { for (const [blockOrdinal, block] of content.entries()) { if (block.type === 'text') { if (block.text.length === 0) { builder.emptyTextBlocks += 1; continue; } builder.addText({ kind: 'text', source_ordinal: sourceOrdinal, block_ordinal: blockOrdinal, role: 'assistant', text: block.text, }); continue; } if (block.type === 'thinking') { builder.addOmission( sourceOrdinal, blockOrdinal, 'assistant_thinking', Buffer.from(block.thinking, 'utf8'), ); continue; } if (block.type === 'toolCall') { builder.addOmission( sourceOrdinal, blockOrdinal, 'tool_call', Buffer.from(canonicalJson(block.arguments), 'utf8'), { toolName: block.name, toolCallId: block.id }, ); continue; } throw unsupportedBlock(`assistant block ${String(Reflect.get(block, 'type'))}`); } } function projectToolResultMessage( builder: ProjectionBuilder, message: Extract, sourceOrdinal: number, ): void { const content = message.content; if (typeof content === 'string') { builder.addOmission(sourceOrdinal, 0, 'tool_result_text', Buffer.from(content, 'utf8'), { toolName: message.toolName, toolCallId: message.toolCallId, }); return; } for (const [blockOrdinal, block] of content.entries()) { if (block.type === 'text') { builder.addOmission( sourceOrdinal, blockOrdinal, 'tool_result_text', Buffer.from(block.text, 'utf8'), { toolName: message.toolName, toolCallId: message.toolCallId }, ); continue; } if (block.type === 'image') { builder.addOmission( sourceOrdinal, blockOrdinal, 'tool_result_image', Buffer.from(block.data, 'utf8'), { toolName: message.toolName, toolCallId: message.toolCallId, mimeType: block.mimeType, }, ); continue; } throw unsupportedBlock(`tool result block ${String(Reflect.get(block, 'type'))}`); } } /** * Deterministic conversation projection. Every retained source block receives * exactly one disposition: included verbatim, or represented as an omission * ledger row. Unknown block types fail loudly instead of disappearing. */ export function projectVisibleConversationV2( messages: readonly Message[], ): ProjectedConversationV2 { const builder = new ProjectionBuilder(); for (const [sourceOrdinal, message] of messages.entries()) { if (message.role === 'user') projectUserMessage(builder, message.content, sourceOrdinal); else if (message.role === 'assistant') projectAssistantMessage(builder, message.content, sourceOrdinal); else if (message.role === 'toolResult') projectToolResultMessage(builder, message, sourceOrdinal); else throw unsupportedBlock(`message role ${String(Reflect.get(message, 'role'))}`); } return builder.finish(messages.length); }