import { type UIMessage as AIMessage, type AssistantContent, type ModelMessage, type DataContent, type FilePart, type GenerateObjectResult, type ImagePart, type StepResult, type ToolContent, type ToolSet, type UserContent, type FileUIPart, type LanguageModelUsage, type CallWarning, type TextPart, type ToolCallPart, type ToolResultPart, type ProviderMetadata, type JSONValue, } from "ai"; import { vMessageWithMetadata, type vSourcePart, type Message, type MessageWithMetadata, type Usage, type vFilePart, type vImagePart, type vReasoningPart, type vRedactedReasoningPart, type vTextPart, type vToolCallPart, type vToolResultPart, type SourcePart, vToolResultOutput, type MessageDoc, vToolApprovalRequest, vToolApprovalResponse, } from "./validators.js"; import type { ActionCtx, AgentComponent } from "./client/types.js"; import type { MutationCtx } from "./client/types.js"; import { MAX_FILE_SIZE, storeFile } from "./client/files.js"; import type { Infer } from "convex/values"; import { convertUint8ArrayToBase64, type ProviderOptions, type ReasoningPart, } from "@ai-sdk/provider-utils"; import { parse, validate } from "convex-helpers/validators"; import { getModelName, getProviderName, type ModelOrMetadata, } from "./shared.js"; export type AIMessageWithoutId = Omit; export type SerializeUrlsAndUint8Arrays = T extends URL ? string : T extends Uint8Array | ArrayBufferLike ? ArrayBuffer : T extends Array ? Array> : T extends Record ? { [K in keyof T]: SerializeUrlsAndUint8Arrays } : T; export type Content = UserContent | AssistantContent | ToolContent; export type SerializedContent = Message["content"]; export type SerializedMessage = Message; export async function serializeMessage( ctx: ActionCtx | MutationCtx, component: AgentComponent, message: ModelMessage | Message, ): Promise<{ message: SerializedMessage; fileIds?: string[] }> { const { content, fileIds } = await serializeContent( ctx, component, message.content, ); return { message: { role: message.role, content, ...(message.providerOptions ? { providerOptions: message.providerOptions } : {}), } as SerializedMessage, fileIds, }; } // Similar to serializeMessage, but doesn't save any files and is looser // For use on the frontend / in synchronous environments. export function fromModelMessage(message: ModelMessage): Message { const content = fromModelMessageContent(message.content); return { role: message.role, content, ...(message.providerOptions ? { providerOptions: message.providerOptions } : {}), } as SerializedMessage; } export async function serializeOrThrow( message: ModelMessage | Message, ): Promise { const { content } = await serializeContent( {} as any, {} as any, message.content, ); return { role: message.role, content, ...(message.providerOptions ? { providerOptions: message.providerOptions } : {}), } as SerializedMessage; } export function toModelMessage( message: SerializedMessage | ModelMessage, ): ModelMessage { return { ...message, content: toModelMessageContent(message.content), } as ModelMessage; } export function docsToModelMessages(messages: MessageDoc[]): ModelMessage[] { return messages .map((m) => m.message) .filter((m) => !!m) .filter((m) => !!m.content.length) .map(toModelMessage); } /** * Scan messages for unresolved `tool-approval-request` parts and inject * synthetic `tool-approval-response` denials so that the AI SDK receives * a complete history (every tool-call has a corresponding result or denial). * * This handles the case where a user sends a new message instead of * resolving pending approvals — the old approvals are auto-denied rather * than silently dropped. */ export function autoDenyUnresolvedApprovals( messages: ModelMessage[], ): ModelMessage[] { // Collect all approval requests: approvalId → { toolCallId, messageIndex } const requests = new Map< string, { toolCallId: string; messageIndex: number } >(); // Collect all resolved approval IDs const resolvedIds = new Set(); for (let i = 0; i < messages.length; i++) { const msg = messages[i]; if (!Array.isArray(msg.content)) continue; for (const part of msg.content as any[]) { if (part.type === "tool-approval-request") { requests.set(part.approvalId, { toolCallId: part.toolCallId, messageIndex: i, }); } else if (part.type === "tool-approval-response") { resolvedIds.add(part.approvalId); } } } // Find unresolved approvals const unresolved: Array<{ approvalId: string; toolCallId: string; messageIndex: number; }> = []; for (const [approvalId, info] of requests) { if (!resolvedIds.has(approvalId)) { unresolved.push({ approvalId, ...info }); } } if (unresolved.length === 0) { return messages; } // Group unresolved approvals by the assistant message index they came from const byMessageIndex = new Map< number, Array<{ approvalId: string; toolCallId: string }> >(); for (const entry of unresolved) { console.warn( `Auto-denying unresolved tool approval ${entry.approvalId} ` + `(toolCallId: ${entry.toolCallId}): new generation started`, ); let group = byMessageIndex.get(entry.messageIndex); if (!group) { group = []; byMessageIndex.set(entry.messageIndex, group); } group.push(entry); } // Build result by inserting synthetic denial messages after each relevant // assistant message const result: ModelMessage[] = []; for (let i = 0; i < messages.length; i++) { result.push(messages[i]); const group = byMessageIndex.get(i); if (group) { result.push({ role: "tool", content: group.map((entry) => ({ type: "tool-approval-response" as const, approvalId: entry.approvalId, approved: false, reason: "auto-denied: new generation started", })), }); } } return result; } export function serializeUsage(usage: LanguageModelUsage): Usage { return { promptTokens: usage.inputTokens ?? 0, completionTokens: usage.outputTokens ?? 0, totalTokens: usage.totalTokens ?? 0, reasoningTokens: usage.reasoningTokens, cachedInputTokens: usage.cachedInputTokens, }; } export function toModelMessageUsage(usage: Usage): LanguageModelUsage { return { inputTokens: usage.promptTokens, outputTokens: usage.completionTokens, totalTokens: usage.totalTokens, reasoningTokens: usage.reasoningTokens, cachedInputTokens: usage.cachedInputTokens, // These detail fields are required by LanguageModelUsage type but we don't // have the granular data, so we provide empty objects with undefined values. inputTokenDetails: { cacheReadTokens: undefined, cacheWriteTokens: undefined, noCacheTokens: undefined, }, outputTokenDetails: { textTokens: undefined, reasoningTokens: undefined, }, }; } export function serializeWarnings( warnings: CallWarning[] | undefined, ): MessageWithMetadata["warnings"] { if (!warnings) { return undefined; } return warnings.map((warning) => { if (warning.type === "compatibility") { return { type: "unsupported-setting", setting: warning.feature, details: warning.details, }; } return warning; }) as any; } export function toModelMessageWarnings( warnings: MessageWithMetadata["warnings"], ): CallWarning[] | undefined { if (!warnings) { return undefined; } return warnings.map((warning) => { if (warning.type === "unsupported-setting") { return { type: "compatibility", feature: warning.setting, details: warning.details, }; } return warning; }) as any; } /** * Serialize explicitly provided response messages for a step. * Used by the streaming/generation loop where the caller tracks which * messages are new via slicing. */ export async function serializeResponseMessages( ctx: ActionCtx, component: AgentComponent, step: StepResult, model: ModelOrMetadata | undefined, responseMessages: ModelMessage[], ): Promise<{ messages: MessageWithMetadata[] }> { return serializeStepMessages(ctx, component, step, model, responseMessages); } /** * Serialize the new messages from a step using a heuristic to determine * which response messages are new (last 1-2 messages). */ export async function serializeNewMessagesInStep( ctx: ActionCtx, component: AgentComponent, step: StepResult, model: ModelOrMetadata | undefined, ): Promise<{ messages: MessageWithMetadata[] }> { const hasToolMessage = step.response.messages.at(-1)?.role === "tool"; let messagesToSerialize: ModelMessage[]; if (hasToolMessage) { messagesToSerialize = step.response.messages.slice(-2); } else if (step.content.length) { messagesToSerialize = step.response.messages.slice(-1); } else { messagesToSerialize = [{ role: "assistant" as const, content: [] }]; } return serializeStepMessages(ctx, component, step, model, messagesToSerialize); } async function serializeStepMessages( ctx: ActionCtx, component: AgentComponent, step: StepResult, model: ModelOrMetadata | undefined, messagesToSerialize: ModelMessage[], ): Promise<{ messages: MessageWithMetadata[] }> { // If there are tool results, there's another message with the tool results // ref: https://github.com/vercel/ai/blob/main/packages/ai/src/generate-text/to-response-messages.ts#L120 const hasToolMessage = step.response.messages.at(-1)?.role === "tool"; const assistantFields = { model: model ? getModelName(model) : undefined, provider: model ? getProviderName(model) : undefined, providerMetadata: step.providerMetadata, reasoning: step.reasoningText, reasoningDetails: step.reasoning, usage: serializeUsage(step.usage), warnings: serializeWarnings(step.warnings), finishReason: step.finishReason, // Only store the sources on one message sources: hasToolMessage ? undefined : step.sources, } satisfies Omit; const toolFields = { sources: step.sources }; const messages: MessageWithMetadata[] = await Promise.all( messagesToSerialize.map(async (msg): Promise => { const { message, fileIds } = await serializeMessage(ctx, component, msg); return parse(vMessageWithMetadata, { message, ...(message.role === "tool" ? toolFields : assistantFields), text: step.text, fileIds, }); }), ); // TODO: capture step.files separately? return { messages }; } export async function serializeObjectResult( ctx: ActionCtx, component: AgentComponent, result: GenerateObjectResult, model: ModelOrMetadata | undefined, ): Promise<{ messages: MessageWithMetadata[] }> { const text = JSON.stringify(result.object); const { message, fileIds } = await serializeMessage(ctx, component, { role: "assistant" as const, content: text, }); return { messages: [ { message, model: model ? getModelName(model) : undefined, provider: model ? getProviderName(model) : undefined, providerMetadata: result.providerMetadata, finishReason: result.finishReason, text, usage: serializeUsage(result.usage), warnings: serializeWarnings(result.warnings), fileIds, }, ], }; } function getMimeOrMediaType(part: { mediaType?: string; mimeType?: string }) { if ("mediaType" in part) { return part.mediaType; } if ("mimeType" in part) { return part.mimeType; } return undefined; } export async function serializeContent( ctx: ActionCtx | MutationCtx, component: AgentComponent, content: Content | Message["content"], ): Promise<{ content: SerializedContent; fileIds?: string[] }> { if (typeof content === "string") { return { content }; } const fileIds: string[] = []; const serialized = await Promise.all( content.map(async (part) => { const metadata: { providerOptions?: ProviderOptions; providerMetadata?: ProviderMetadata; } = {}; if ("providerOptions" in part) { metadata.providerOptions = part.providerOptions as ProviderOptions; } if ("providerMetadata" in part) { metadata.providerMetadata = part.providerMetadata as ProviderMetadata; } switch (part.type) { case "text": { return { type: part.type, text: part.text, ...metadata, } satisfies Infer; } case "image": { let image = serializeDataOrUrl(part.image); if ( image instanceof ArrayBuffer && image.byteLength > MAX_FILE_SIZE ) { const { file } = await storeFile( ctx, component, new Blob([image], { type: getMimeOrMediaType(part) || guessMimeType(image), }), ); image = file.url; fileIds.push(file.fileId); } return { type: part.type, mediaType: getMimeOrMediaType(part), ...metadata, image, } satisfies Infer; } case "file": { let data = serializeDataOrUrl(part.data); if (data instanceof ArrayBuffer && data.byteLength > MAX_FILE_SIZE) { const { file } = await storeFile( ctx, component, new Blob([data], { type: getMimeOrMediaType(part) }), ); data = file.url; fileIds.push(file.fileId); } return { type: part.type, data, filename: part.filename, mediaType: getMimeOrMediaType(part)!, ...metadata, } satisfies Infer; } case "tool-call": { // Handle legacy data where only args field exists const input = part.input ?? (part as any)?.args ?? {}; return { type: part.type, input, /** @deprecated Use `input` instead. */ args: input, toolCallId: part.toolCallId, toolName: part.toolName, providerExecuted: part.providerExecuted, ...metadata, } satisfies Infer; } case "tool-result": { return normalizeToolResult(part, metadata); } case "reasoning": { return { type: part.type, text: part.text, ...metadata, } satisfies Infer; } // Not in current generation output, but could be in historical messages case "redacted-reasoning": { return { type: part.type, data: part.data, ...metadata, } satisfies Infer; } case "source": { return part satisfies Infer; } case "tool-approval-request": { return { type: part.type, approvalId: part.approvalId, toolCallId: part.toolCallId, ...metadata, } satisfies Infer; } case "tool-approval-response": { return { type: part.type, approvalId: part.approvalId, approved: part.approved, reason: part.reason, providerExecuted: part.providerExecuted, ...metadata, } satisfies Infer; } default: return null; } }), ); return { content: serialized.filter((p) => p !== null) as SerializedContent, fileIds: fileIds.length > 0 ? fileIds : undefined, }; } export function fromModelMessageContent(content: Content): Message["content"] { if (typeof content === "string") { return content; } return content .map((part) => { const metadata: { providerOptions?: ProviderOptions; providerMetadata?: ProviderMetadata; } = {}; if ("providerOptions" in part) { metadata.providerOptions = part.providerOptions as ProviderOptions; } if ("providerMetadata" in part) { metadata.providerMetadata = part.providerMetadata as ProviderMetadata; } switch (part.type) { case "text": return part satisfies Infer; case "image": return { type: part.type, mediaType: getMimeOrMediaType(part), ...metadata, image: serializeDataOrUrl(part.image), } satisfies Infer; case "file": return { type: part.type, data: serializeDataOrUrl(part.data), filename: part.filename, mediaType: getMimeOrMediaType(part)!, ...metadata, } satisfies Infer; case "tool-call": // Handle legacy data where only args field exists return { type: part.type, input: part.input ?? (part as any)?.args ?? {}, /** @deprecated Use `input` instead. */ args: part.input ?? (part as any)?.args ?? {}, toolCallId: part.toolCallId, toolName: part.toolName, providerExecuted: part.providerExecuted, ...metadata, } satisfies Infer; case "tool-result": return normalizeToolResult(part, metadata); case "reasoning": return { type: part.type, text: part.text, ...metadata, } satisfies Infer; case "tool-approval-request": return { type: part.type, approvalId: part.approvalId, toolCallId: part.toolCallId, ...metadata, } satisfies Infer; case "tool-approval-response": return { type: part.type, approvalId: part.approvalId, approved: part.approved, reason: part.reason, providerExecuted: part.providerExecuted, ...metadata, } satisfies Infer; // Not in current generation output, but could be in historical messages default: return null; } }) .filter((p) => p !== null) as Message["content"]; } export function toModelMessageContent( content: SerializedContent | ModelMessage["content"], ): Content { if (typeof content === "string") { return content; } return content .map((part) => { const metadata: { providerOptions?: ProviderOptions; providerMetadata?: ProviderMetadata; } = {}; if ("providerOptions" in part) { metadata.providerOptions = part.providerOptions; } if ("providerMetadata" in part) { metadata.providerMetadata = part.providerMetadata; } switch (part.type) { case "text": return { type: part.type, text: part.text, ...metadata, } satisfies TextPart; case "image": return { type: part.type, image: toModelMessageDataOrUrl(part.image), mediaType: getMimeOrMediaType(part), ...metadata, } satisfies ImagePart; case "file": return { type: part.type, data: toModelMessageDataOrUrl(part.data), filename: part.filename, mediaType: getMimeOrMediaType(part)!, ...metadata, } satisfies FilePart; case "tool-call": { // Handle legacy data where only args field exists const input = part.input ?? (part as any)?.args ?? {}; return { type: part.type, input, toolCallId: part.toolCallId, toolName: part.toolName, providerExecuted: part.providerExecuted, ...metadata, } satisfies ToolCallPart; } case "tool-result": { return normalizeToolResult(part, metadata); } case "reasoning": return { type: part.type, text: part.text, ...metadata, } satisfies ReasoningPart; case "redacted-reasoning": // TODO: should we just drop this? return { type: "reasoning", text: "", ...metadata, providerOptions: metadata.providerOptions ? { ...Object.fromEntries( Object.entries(metadata.providerOptions ?? {}).map( ([key, value]) => [ key, { ...value, redactedData: part.data }, ], ), ), } : undefined, } satisfies ReasoningPart; case "source": return part satisfies SourcePart; case "tool-approval-request": return { type: part.type, approvalId: part.approvalId, toolCallId: part.toolCallId, ...metadata, } satisfies Infer; case "tool-approval-response": return { type: part.type, approvalId: part.approvalId, approved: part.approved, reason: part.reason, providerExecuted: part.providerExecuted, ...metadata, } satisfies Infer; default: return null; } }) .filter((p) => p !== null) as Content; } export function normalizeToolOutput( result: string | JSONValue | undefined, ): ToolResultPart["output"] { if (typeof result === "string") { return { type: "text", value: result, }; } if (validate(vToolResultOutput, result)) { return result; } return { type: "json", value: result ?? null, }; } function normalizeToolResult( part: ToolResultPart | Infer, metadata: { providerOptions?: ProviderOptions; providerMetadata?: ProviderMetadata; }, ): ToolResultPart & Infer { return { type: part.type, output: part.output ? validate(vToolResultOutput, part.output) ? (part.output as any) : normalizeToolOutput(JSON.stringify(part.output)) : normalizeToolOutput("result" in part ? part.result : undefined), toolCallId: part.toolCallId, toolName: part.toolName, // Preserve isError flag for error reporting ...("isError" in part && part.isError ? { isError: true } : {}), ...metadata, } satisfies ToolResultPart; } /** * Return a best-guess MIME type based on the magic-number signature * found at the start of an ArrayBuffer. * * @param buf – the source ArrayBuffer * @returns the detected MIME type, or `"application/octet-stream"` if unknown */ export function guessMimeType(buf: ArrayBuffer | string): string { if (typeof buf === "string") { if (buf.match(/^data:\w+\/\w+;base64/)) { return buf.split(";")[0].split(":")[1]!; } return "text/plain"; } if (buf.byteLength < 4) return "application/octet-stream"; // Read the first 12 bytes (enough for all signatures below) const bytes = new Uint8Array(buf.slice(0, 12)); const hex = [...bytes].map((b) => b.toString(16).padStart(2, "0")).join(""); // Helper so we can look at only the needed prefix const startsWith = (sig: string) => hex.startsWith(sig.toLowerCase()); // --- image formats --- if (startsWith("89504e47")) return "image/png"; // PNG - 89 50 4E 47 if ( startsWith("ffd8ffdb") || startsWith("ffd8ffe0") || startsWith("ffd8ffee") || startsWith("ffd8ffe1") ) return "image/jpeg"; // JPEG if (startsWith("47494638")) return "image/gif"; // GIF if (startsWith("424d")) return "image/bmp"; // BMP if (startsWith("52494646") && hex.substr(16, 8) === "57454250") return "image/webp"; // WEBP (RIFF....WEBP) if (startsWith("49492a00")) return "image/tiff"; // TIFF // typeof m === "object" && m !== null && "parts" in m, // ) // ) { // messages.push(...convertToModelMessages(args.messages)); // } else { // messages.push(...modelMessageSchema.array().parse(args.messages)); // } // } // return messages; // }