{"version":3,"file":"utils/multimodal.mjs","sources":["webpack://@multimodal/agent/./src/utils/multimodal.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\n\nimport type {\n  ChatCompletionContentPart,\n  MultimodalToolCallResult,\n  ToolCallResult,\n} from '@multimodal/agent-interface';\nimport { getLogger } from './logger';\n\nconst logger = getLogger('Multimodal');\n\n/**\n * Try to extract image data from an object\n *\n * @param obj The object to check for image data\n * @returns The image data info if found, null otherwise\n */\nfunction extractImageData(obj: any): { data: string; mimeType: string } | null {\n  // Object explicitly defines image type\n  if (obj.type && obj.data && typeof obj.data === 'string') {\n    // If object itself indicates image type\n    if (obj.type.includes('screenshot') || obj.type.includes('image')) {\n      // If object specifies mime type\n      if (obj.mimeType && typeof obj.mimeType === 'string') {\n        return { data: obj.data, mimeType: obj.mimeType };\n      }\n\n      // Try to detect format from type field\n      if (obj.type.includes('png')) {\n        return { data: obj.data, mimeType: 'image/png' };\n      } else if (obj.type.includes('jpg') || obj.type.includes('jpeg')) {\n        return { data: obj.data, mimeType: 'image/jpeg' };\n      } else if (obj.type.includes('gif')) {\n        return { data: obj.data, mimeType: 'image/gif' };\n      } else if (obj.type.includes('webp')) {\n        return { data: obj.data, mimeType: 'image/webp' };\n      }\n\n      // Default to PNG if we can't determine from type\n      return { data: obj.data, mimeType: 'image/png' };\n    }\n  }\n\n  // Detect by common base64 image patterns\n  if (obj.data && typeof obj.data === 'string') {\n    const data = obj.data.trim();\n    // Check if it looks like valid base64 data\n    if (data.match(/^[A-Za-z0-9+/=]+$/)) {\n      if (data.startsWith('iVBOR')) {\n        // PNG signature\n        return { data, mimeType: 'image/png' };\n      } else if (data.startsWith('/9j/')) {\n        // JPEG signature\n        return { data, mimeType: 'image/jpeg' };\n      } else if (data.startsWith('R0lGOD')) {\n        // GIF signature\n        return { data, mimeType: 'image/gif' };\n      } else if (data.startsWith('UklGR')) {\n        // WEBP signature\n        return { data, mimeType: 'image/webp' };\n      } else {\n        // Generic base64, but still looks valid\n        // Default to PNG as a fallback\n        return { data, mimeType: 'image/png' };\n      }\n    }\n  }\n\n  return null;\n}\n\n/**\n * Convert a ToolCallResult to MultimodalToolCallResult\n *\n * This function transforms the original tool call result into a multimodal format\n * that can be processed by LLMs supporting multimodal inputs.\n *\n * Handled cases:\n * - String content: Converted to text type content part\n * - Array content:\n *   - Arrays containing objects with image data: Extracts images and preserves other data\n *   - Arrays of primitive values or objects without images: Stringified and converted to text\n * - Object content with image data: Extracts image data and adds as image_url type\n *   - Supports common image formats (PNG, JPEG, GIF, WEBP)\n *   - Detects image data through explicit type declarations or base64 signatures\n *   - Preserves non-image data from the object as additional text content\n * - Plain objects without image data: Stringified and converted to text type\n * - Null/undefined values: Converted to empty string\n *\n * Not handled cases:\n * - Nested multimodal content beyond one level (images in nested objects beyond first level)\n * - Binary image formats that aren't base64 encoded\n * - SVG or other vector image formats\n * - Video, audio or other non-image media types\n * - Complex hierarchical data structures that might benefit from custom formatting\n *\n * Error handling:\n * - Image processing errors default back to text representation\n * - Any unexpected errors during conversion result in an error message text\n *\n * @param toolCallResult The original tool call result\n * @returns The multimodal version of the tool call result containing text and/or image content parts\n */\nexport function convertToMultimodalToolCallResult(\n  toolCallResult: ToolCallResult,\n): MultimodalToolCallResult {\n  const { toolCallId, toolName, content } = toolCallResult;\n  const contentParts: ChatCompletionContentPart[] = [];\n\n  try {\n    // Handle string content\n    if (typeof content === 'string') {\n      contentParts.push({\n        type: 'text',\n        text: content,\n      });\n    }\n    // Handle array content\n    else if (Array.isArray(content)) {\n      // Check if the array contains objects with image data\n      let hasImageInArray = false;\n      const processedContent = [];\n\n      for (const item of content) {\n        // Only process object elements in the array, skip other types\n        if (typeof item === 'object' && item !== null) {\n          const imageInfo = extractImageData(item);\n\n          if (imageInfo) {\n            hasImageInArray = true;\n            // Add image part\n            const imageSize = imageInfo.data.length;\n            logger.debug(`Converting image data (size: ${imageSize} bytes) to image_url format`);\n\n            contentParts.push({\n              type: 'image_url',\n              image_url: { url: `data:${imageInfo.mimeType};base64,${imageInfo.data}` },\n            });\n\n            // Extract other data from the object\n            const remainingContent = { ...item };\n            delete remainingContent.data;\n            delete remainingContent.type;\n            delete remainingContent.mimeType;\n\n            // Keep other properties in processed content\n            if (Object.keys(remainingContent).length > 0) {\n              processedContent.push(remainingContent);\n            }\n          } else {\n            // No image in object, keep complete object\n            processedContent.push(item);\n          }\n        } else {\n          // Non-object elements are preserved directly\n          processedContent.push(item);\n        }\n      }\n\n      // If there are processed content items, add them as text\n      if (processedContent.length > 0) {\n        contentParts.push({\n          type: 'text',\n          text: JSON.stringify(processedContent, null, 2),\n        });\n      }\n\n      // If no image was found in the array, fall back to the original content\n      if (!hasImageInArray && contentParts.length === 0) {\n        contentParts.push({\n          type: 'text',\n          text: JSON.stringify(content, null, 2),\n        });\n      }\n    }\n    // Handle object content\n    else if (typeof content === 'object' && content !== null) {\n      // Check if the object contains image data\n      const imageInfo = extractImageData(content);\n\n      if (imageInfo) {\n        try {\n          // Add the image part with proper mime type\n          logger.debug(\n            `Found image data (${imageInfo.mimeType}) of size: ${imageInfo.data.length} bytes`,\n          );\n\n          contentParts.push({\n            type: 'image_url',\n            image_url: { url: `data:${imageInfo.mimeType};base64,${imageInfo.data}` },\n          });\n\n          // Add remaining data as text if there's more content besides the image data\n          const remainingContent = { ...content };\n          // Remove image-related properties\n          delete remainingContent.data;\n          delete remainingContent.type;\n          delete remainingContent.mimeType;\n\n          // Only stringify and add remaining content if it's not empty\n          if (Object.keys(remainingContent).length > 0) {\n            contentParts.push({\n              type: 'text',\n              text: JSON.stringify(remainingContent, null, 2),\n            });\n          }\n        } catch (error) {\n          logger.warn(`Failed to process image data: ${error}`);\n          // If there's an error with the image data, fall back to text representation\n          contentParts.push({\n            type: 'text',\n            text: JSON.stringify(content, null, 2),\n          });\n        }\n      } else {\n        // If no image data, convert the entire object to a text string\n        contentParts.push({\n          type: 'text',\n          text: JSON.stringify(content, null, 2),\n        });\n      }\n    }\n    // Handle null, undefined or other types\n    else {\n      contentParts.push({\n        type: 'text',\n        text: String(content),\n      });\n    }\n\n    // Ensure we have at least one content part\n    if (contentParts.length === 0) {\n      contentParts.push({\n        type: 'text',\n        text: '',\n      });\n    }\n\n    // 记录创建了多少个内容部分，但不输出具体内容\n    logger.debug(`Created ${contentParts.length} content parts for tool call result`);\n  } catch (error) {\n    // Fallback for any unexpected errors\n    logger.error(`Error in convertToMultimodalToolCallResult: ${error}`);\n    contentParts.push({\n      type: 'text',\n      text: `Error processing content: ${String(error)}`,\n    });\n  }\n\n  return {\n    toolCallId,\n    toolName,\n    content: contentParts,\n  };\n}\n"],"names":["logger","getLogger","extractImageData","obj","data","convertToMultimodalToolCallResult","toolCallResult","toolCallId","toolName","content","contentParts","Array","hasImageInArray","processedContent","item","imageInfo","imageSize","remainingContent","Object","JSON","error","String"],"mappings":";;;;;AAaA,MAAMA,SAASC,UAAU;AAQzB,SAASC,iBAAiBC,GAAQ;IAEhC,IAAIA,IAAI,IAAI,IAAIA,IAAI,IAAI,IAAI,AAAoB,YAApB,OAAOA,IAAI,IAAI,EAEzC;QAAA,IAAIA,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiBA,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU;YAEjE,IAAIA,IAAI,QAAQ,IAAI,AAAwB,YAAxB,OAAOA,IAAI,QAAQ,EACrC,OAAO;gBAAE,MAAMA,IAAI,IAAI;gBAAE,UAAUA,IAAI,QAAQ;YAAC;YAIlD,IAAIA,IAAI,IAAI,CAAC,QAAQ,CAAC;iBAEf,IAAIA,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAUA,IAAI,IAAI,CAAC,QAAQ,CAAC,SACvD,OAAO;gBAAE,MAAMA,IAAI,IAAI;gBAAE,UAAU;YAAa;iBAC3C,IAAIA,IAAI,IAAI,CAAC,QAAQ,CAAC,QAC3B,OAAO;gBAAE,MAAMA,IAAI,IAAI;gBAAE,UAAU;YAAY;iBAC1C,IAAIA,IAAI,IAAI,CAAC,QAAQ,CAAC,SAC3B,OAAO;gBAAE,MAAMA,IAAI,IAAI;gBAAE,UAAU;YAAa;YAIlD,OAAO;gBAAE,MAAMA,IAAI,IAAI;gBAAE,UAAU;YAAY;QACjD;IAAA;IAIF,IAAIA,IAAI,IAAI,IAAI,AAAoB,YAApB,OAAOA,IAAI,IAAI,EAAe;QAC5C,MAAMC,OAAOD,IAAI,IAAI,CAAC,IAAI;QAE1B,IAAIC,KAAK,KAAK,CAAC,sBACb,IAAIA,KAAK,UAAU,CAAC,UAElB,OAAO;YAAEA;YAAM,UAAU;QAAY;aAChC,IAAIA,KAAK,UAAU,CAAC,SAEzB,OAAO;YAAEA;YAAM,UAAU;QAAa;aACjC,IAAIA,KAAK,UAAU,CAAC,WAEzB,OAAO;YAAEA;YAAM,UAAU;QAAY;aAChC,IAAIA,KAAK,UAAU,CAAC,UAEzB,OAAO;YAAEA;YAAM,UAAU;QAAa;aAItC,OAAO;YAAEA;YAAM,UAAU;QAAY;IAG3C;IAEA,OAAO;AACT;AAkCO,SAASC,kCACdC,cAA8B;IAE9B,MAAM,EAAEC,UAAU,EAAEC,QAAQ,EAAEC,OAAO,EAAE,GAAGH;IAC1C,MAAMI,eAA4C,EAAE;IAEpD,IAAI;QAEF,IAAI,AAAmB,YAAnB,OAAOD,SACTC,aAAa,IAAI,CAAC;YAChB,MAAM;YACN,MAAMD;QACR;aAGG,IAAIE,MAAM,OAAO,CAACF,UAAU;YAE/B,IAAIG,kBAAkB;YACtB,MAAMC,mBAAmB,EAAE;YAE3B,KAAK,MAAMC,QAAQL,QAEjB,IAAI,AAAgB,YAAhB,OAAOK,QAAqBA,AAAS,SAATA,MAAe;gBAC7C,MAAMC,YAAYb,iBAAiBY;gBAEnC,IAAIC,WAAW;oBACbH,kBAAkB;oBAElB,MAAMI,YAAYD,UAAU,IAAI,CAAC,MAAM;oBACvCf,OAAO,KAAK,CAAC,CAAC,6BAA6B,EAAEgB,UAAU,2BAA2B,CAAC;oBAEnFN,aAAa,IAAI,CAAC;wBAChB,MAAM;wBACN,WAAW;4BAAE,KAAK,CAAC,KAAK,EAAEK,UAAU,QAAQ,CAAC,QAAQ,EAAEA,UAAU,IAAI,EAAE;wBAAC;oBAC1E;oBAGA,MAAME,mBAAmB;wBAAE,GAAGH,IAAI;oBAAC;oBACnC,OAAOG,iBAAiB,IAAI;oBAC5B,OAAOA,iBAAiB,IAAI;oBAC5B,OAAOA,iBAAiB,QAAQ;oBAGhC,IAAIC,OAAO,IAAI,CAACD,kBAAkB,MAAM,GAAG,GACzCJ,iBAAiB,IAAI,CAACI;gBAE1B,OAEEJ,iBAAiB,IAAI,CAACC;YAE1B,OAEED,iBAAiB,IAAI,CAACC;YAK1B,IAAID,iBAAiB,MAAM,GAAG,GAC5BH,aAAa,IAAI,CAAC;gBAChB,MAAM;gBACN,MAAMS,KAAK,SAAS,CAACN,kBAAkB,MAAM;YAC/C;YAIF,IAAI,CAACD,mBAAmBF,AAAwB,MAAxBA,aAAa,MAAM,EACzCA,aAAa,IAAI,CAAC;gBAChB,MAAM;gBACN,MAAMS,KAAK,SAAS,CAACV,SAAS,MAAM;YACtC;QAEJ,OAEK,IAAI,AAAmB,YAAnB,OAAOA,WAAwBA,AAAY,SAAZA,SAAkB;YAExD,MAAMM,YAAYb,iBAAiBO;YAEnC,IAAIM,WACF,IAAI;gBAEFf,OAAO,KAAK,CACV,CAAC,kBAAkB,EAAEe,UAAU,QAAQ,CAAC,WAAW,EAAEA,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;gBAGpFL,aAAa,IAAI,CAAC;oBAChB,MAAM;oBACN,WAAW;wBAAE,KAAK,CAAC,KAAK,EAAEK,UAAU,QAAQ,CAAC,QAAQ,EAAEA,UAAU,IAAI,EAAE;oBAAC;gBAC1E;gBAGA,MAAME,mBAAmB;oBAAE,GAAGR,OAAO;gBAAC;gBAEtC,OAAOQ,iBAAiB,IAAI;gBAC5B,OAAOA,iBAAiB,IAAI;gBAC5B,OAAOA,iBAAiB,QAAQ;gBAGhC,IAAIC,OAAO,IAAI,CAACD,kBAAkB,MAAM,GAAG,GACzCP,aAAa,IAAI,CAAC;oBAChB,MAAM;oBACN,MAAMS,KAAK,SAAS,CAACF,kBAAkB,MAAM;gBAC/C;YAEJ,EAAE,OAAOG,OAAO;gBACdpB,OAAO,IAAI,CAAC,CAAC,8BAA8B,EAAEoB,OAAO;gBAEpDV,aAAa,IAAI,CAAC;oBAChB,MAAM;oBACN,MAAMS,KAAK,SAAS,CAACV,SAAS,MAAM;gBACtC;YACF;iBAGAC,aAAa,IAAI,CAAC;gBAChB,MAAM;gBACN,MAAMS,KAAK,SAAS,CAACV,SAAS,MAAM;YACtC;QAEJ,OAGEC,aAAa,IAAI,CAAC;YAChB,MAAM;YACN,MAAMW,OAAOZ;QACf;QAIF,IAAIC,AAAwB,MAAxBA,aAAa,MAAM,EACrBA,aAAa,IAAI,CAAC;YAChB,MAAM;YACN,MAAM;QACR;QAIFV,OAAO,KAAK,CAAC,CAAC,QAAQ,EAAEU,aAAa,MAAM,CAAC,mCAAmC,CAAC;IAClF,EAAE,OAAOU,OAAO;QAEdpB,OAAO,KAAK,CAAC,CAAC,4CAA4C,EAAEoB,OAAO;QACnEV,aAAa,IAAI,CAAC;YAChB,MAAM;YACN,MAAM,CAAC,0BAA0B,EAAEW,OAAOD,QAAQ;QACpD;IACF;IAEA,OAAO;QACLb;QACAC;QACA,SAASE;IACX;AACF"}