{"version":3,"file":"GUIAgent.mjs","sources":["webpack://@ui-tars-test/agent-sdk/./src/GUIAgent.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport {\n  LLMRequestHookPayload,\n  ChatCompletionContentPart,\n  LogLevel,\n  Tool,\n  ConsoleLogger,\n} from '@ui-tars-test/tarko-agent';\nimport { GUIAgentToolCallEngine } from './ToolCallEngine';\nimport { SYSTEM_PROMPT } from './prompts';\nimport { Base64ImageParser } from '@agent-infra/media-utils';\nimport { Operator, BaseGUIAgent } from '@ui-tars-test/shared/base';\nimport {\n  GUIAgentConfig,\n  NormalizeCoordinates,\n  ImageDetailCalculator,\n} from '@ui-tars-test/shared/types';\nimport {\n  assembleSystemPrompt,\n  isSystemPromptTemplate,\n  normalizeActionCoords,\n  sleep,\n} from '@ui-tars-test/shared/utils';\nimport { GUI_ADAPTED_TOOL_NAME } from './constants';\nimport { convertToAgentUIAction, createGUIErrorResponse } from './utils';\nimport { defaultNormalizeCoords, defaultDetailCalculator } from './defaultImpls';\n\nconst defaultLogger = new ConsoleLogger('[GUIAgent]', LogLevel.DEBUG);\n\nexport class GUIAgent<T extends Operator> extends BaseGUIAgent {\n  static label = 'GUI Agent';\n\n  private operator: Operator | undefined;\n  private normalizeCoordinates: NormalizeCoordinates;\n  private detailCalculator: ImageDetailCalculator;\n  private loopIntervalInMs: number;\n\n  constructor(config: GUIAgentConfig<T>) {\n    const {\n      operator,\n      model,\n      systemPrompt,\n      customeActionParser,\n      normalizeCoordinates,\n      detailCalculator,\n      maxLoopCount,\n      loopIntervalInMs = 500,\n    } = config;\n    let finalSystemPrompt = SYSTEM_PROMPT;\n    if (typeof systemPrompt === 'string') {\n      finalSystemPrompt = systemPrompt;\n    } else if (Array.isArray(systemPrompt)) {\n      finalSystemPrompt = systemPrompt\n        .map((p) => (typeof p === 'string' ? p : p.content))\n        .join('\\n\\n');\n    } else if (systemPrompt && isSystemPromptTemplate(systemPrompt)) {\n      finalSystemPrompt = assembleSystemPrompt(systemPrompt, operator.getSupportedActions());\n    }\n    defaultLogger.debug('final instructions for sp:', finalSystemPrompt);\n\n    // Create a adapted ToolCallEngine constructor that captures the customeActionParser\n    const AdaptedToolCallEngine = class extends GUIAgentToolCallEngine {\n      constructor() {\n        super(customeActionParser);\n      }\n    };\n\n    super({\n      name: GUIAgent.label,\n      instructions: finalSystemPrompt,\n      tools: [],\n      toolCallEngine: AdaptedToolCallEngine,\n      model: model,\n      ...(maxLoopCount && { maxIterations: maxLoopCount }),\n      logLevel: LogLevel.DEBUG,\n    });\n    this.operator = operator;\n    this.normalizeCoordinates = normalizeCoordinates ?? defaultNormalizeCoords;\n    // Default detail calculator implementation\n    this.detailCalculator = detailCalculator ?? defaultDetailCalculator;\n    this.loopIntervalInMs = loopIntervalInMs;\n    this.logger = this.logger.spawn('[GUIAgent]');\n  }\n\n  async initialize() {\n    // Register the GUI tool\n    this.registerTool(\n      new Tool({\n        id: GUI_ADAPTED_TOOL_NAME,\n        description: 'operator tool',\n        parameters: {}, // no need to pass parameters\n        function: async (input) => {\n          this.logger.log(`${GUI_ADAPTED_TOOL_NAME} input:`, input);\n          if (!this.operator) {\n            return createGUIErrorResponse(input.action, 'Operator not initialized');\n          }\n          if (input.errorMessage) {\n            return createGUIErrorResponse(input.action, input.errorMessage);\n          }\n          // normalize coordinates\n          if (input.operator_action) {\n            input.operator_action = normalizeActionCoords(\n              input.operator_action,\n              this.normalizeCoordinates,\n            );\n          }\n          this.logger.info('action to execute:', JSON.stringify(input.operator_action));\n          const result = await this.operator!.doExecute({\n            actions: [input.operator_action],\n          });\n          if (result.errorMessage) {\n            return createGUIErrorResponse(input.action, result.errorMessage);\n          }\n          // return { action: input.action, status: 'success', result };\n          return {\n            success: true,\n            action: input.action,\n            normalizedAction: convertToAgentUIAction(input.operator_action),\n            observation: undefined, // Reserved for future implementation\n          };\n        },\n      }),\n    );\n    super.initialize();\n  }\n\n  async onLLMRequest(id: string, payload: LLMRequestHookPayload): Promise<void> {\n    try {\n      const safeStringify = (obj: unknown, max = 800): string => {\n        try {\n          const s = JSON.stringify(obj);\n          return s.length > max ? s.slice(0, max) + '…(truncated)' : s;\n        } catch {\n          return '[unserializable]';\n        }\n      };\n\n      const req = payload?.request;\n      const messages = req && Array.isArray(req.messages) ? req.messages : [];\n      const model = req && typeof req.model === 'string' ? req.model : 'unknown';\n\n      const hasImages = (() => {\n        try {\n          let cnt = 0;\n          for (const m of messages as Array<{ content?: unknown }>) {\n            const content = m?.content;\n            if (Array.isArray(content)) {\n              for (const part of content as Array<{ type?: string }>) {\n                if (part?.type === 'image_url') cnt++;\n              }\n            }\n          }\n          return cnt;\n        } catch {\n          return undefined as unknown as number | undefined;\n        }\n      })();\n\n      const summary = {\n        id,\n        model,\n        messagesCount: messages.length,\n        hasImages,\n      };\n\n      this.logger.info('[GUIAgent] onLLMRequest summary:', safeStringify(summary));\n\n      const firstMsg = messages[0];\n      if (firstMsg) {\n        this.logger.debug('[GUIAgent] onLLMRequest first message:', safeStringify(firstMsg, 1200));\n      }\n    } catch (e) {\n      this.logger.error('[GUIAgent] onLLMRequest logging failed:', e);\n    }\n  }\n\n  async onEachAgentLoopStart(sessionId: string) {\n    this.logger.info('onEachAgentLoopStart', sessionId);\n  }\n\n  async onAgentLoopEnd(id: string): Promise<void> {\n    // await this.browserOperator.cleanup();\n  }\n\n  async onBeforeToolCall(\n    id: string,\n    toolCall: { toolCallId: string; name: string },\n    args: unknown,\n  ) {\n    return args;\n  }\n\n  async onAfterToolCall(\n    id: string,\n    toolCall: { toolCallId: string; name: string },\n    result: unknown,\n  ): Promise<unknown> {\n    this.logger.info('onAfterToolCall toolCall', JSON.stringify(toolCall));\n\n    if (toolCall.name !== GUI_ADAPTED_TOOL_NAME) {\n      this.logger.info('onAfterToolCall: skipping screenshot');\n      return;\n    }\n\n    await sleep(this.loopIntervalInMs);\n\n    const output = await this.operator!.doScreenshot();\n    if (!output) {\n      this.logger.error('Failed to get screenshot');\n      return;\n    }\n\n    const base64Tool = new Base64ImageParser(output.base64);\n    const base64Uri = base64Tool.getDataUri();\n    if (!base64Uri) {\n      this.logger.error('Failed to get base64 image uri');\n      return;\n    }\n\n    const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() || {\n      width: -1,\n      height: -1,\n    };\n\n    const content: ChatCompletionContentPart[] = [\n      {\n        type: 'image_url',\n        image_url: {\n          url: base64Uri,\n          detail: this.detailCalculator(imageWidth, imageHeight),\n        },\n      },\n      {\n        type: 'text',\n        text: `Screen resolution: ${imageWidth}x${imageHeight}. System is running on macOS. Please return actions with absolute coordinates within this resolution.`,\n      },\n    ];\n\n    if (output?.url) {\n      content.push({\n        type: 'text',\n        text: `The current page's url: ${output.url}`,\n      });\n    }\n\n    const eventStream = this.getEventStream();\n    const events = eventStream.getEvents();\n    this.logger.info('onAfterToolCall events length:', events.length);\n\n    const event = eventStream.createEvent('environment_input', {\n      description: 'Browser Screenshot',\n      content,\n      metadata: {\n        type: 'screenshot',\n        url: output?.url,\n      },\n    });\n    eventStream.sendEvent(event);\n    return result;\n  }\n}\n"],"names":["defaultLogger","ConsoleLogger","LogLevel","GUIAgent","BaseGUIAgent","Tool","GUI_ADAPTED_TOOL_NAME","input","createGUIErrorResponse","normalizeActionCoords","JSON","result","convertToAgentUIAction","undefined","id","payload","safeStringify","obj","max","s","req","messages","Array","model","hasImages","cnt","m","content","part","summary","firstMsg","e","sessionId","toolCall","args","sleep","output","base64Tool","Base64ImageParser","base64Uri","imageWidth","imageHeight","eventStream","events","event","config","operator","systemPrompt","customeActionParser","normalizeCoordinates","detailCalculator","maxLoopCount","loopIntervalInMs","finalSystemPrompt","SYSTEM_PROMPT","p","isSystemPromptTemplate","assembleSystemPrompt","AdaptedToolCallEngine","GUIAgentToolCallEngine","defaultNormalizeCoords","defaultDetailCalculator"],"mappings":";;;;;;;;;;;;;AAGC;;;;;;;;;;AA2BD,MAAMA,gBAAgB,IAAIC,cAAc,cAAcC,SAAS,KAAK;AAE7D,MAAMC,iBAAqCC;IAuDhD,MAAM,aAAa;QAEjB,IAAI,CAAC,YAAY,CACf,IAAIC,KAAK;YACP,IAAIC;YACJ,aAAa;YACb,YAAY,CAAC;YACb,UAAU,OAAOC;gBACf,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAGD,sBAAsB,OAAO,CAAC,EAAEC;gBACnD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAChB,OAAOC,uBAAuBD,MAAM,MAAM,EAAE;gBAE9C,IAAIA,MAAM,YAAY,EACpB,OAAOC,uBAAuBD,MAAM,MAAM,EAAEA,MAAM,YAAY;gBAGhE,IAAIA,MAAM,eAAe,EACvBA,MAAM,eAAe,GAAGE,sBACtBF,MAAM,eAAe,EACrB,IAAI,CAAC,oBAAoB;gBAG7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsBG,KAAK,SAAS,CAACH,MAAM,eAAe;gBAC3E,MAAMI,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,SAAS,CAAC;oBAC5C,SAAS;wBAACJ,MAAM,eAAe;qBAAC;gBAClC;gBACA,IAAII,OAAO,YAAY,EACrB,OAAOH,uBAAuBD,MAAM,MAAM,EAAEI,OAAO,YAAY;gBAGjE,OAAO;oBACL,SAAS;oBACT,QAAQJ,MAAM,MAAM;oBACpB,kBAAkBK,uBAAuBL,MAAM,eAAe;oBAC9D,aAAaM;gBACf;YACF;QACF;QAEF,KAAK,CAAC;IACR;IAEA,MAAM,aAAaC,EAAU,EAAEC,OAA8B,EAAiB;QAC5E,IAAI;YACF,MAAMC,gBAAgB,CAACC,KAAcC,MAAM,GAAG;gBAC5C,IAAI;oBACF,MAAMC,IAAIT,KAAK,SAAS,CAACO;oBACzB,OAAOE,EAAE,MAAM,GAAGD,MAAMC,EAAE,KAAK,CAAC,GAAGD,OAAO,sBAAiBC;gBAC7D,EAAE,OAAM;oBACN,OAAO;gBACT;YACF;YAEA,MAAMC,MAAML,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO;YAC5B,MAAMM,WAAWD,OAAOE,MAAM,OAAO,CAACF,IAAI,QAAQ,IAAIA,IAAI,QAAQ,GAAG,EAAE;YACvE,MAAMG,QAAQH,OAAO,AAAqB,YAArB,OAAOA,IAAI,KAAK,GAAgBA,IAAI,KAAK,GAAG;YAEjE,MAAMI,YAAa,AAAC;gBAClB,IAAI;oBACF,IAAIC,MAAM;oBACV,KAAK,MAAMC,KAAKL,SAA0C;wBACxD,MAAMM,UAAUD,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO;wBAC1B,IAAIJ,MAAM,OAAO,CAACK,UAChB;4BAAA,KAAK,MAAMC,QAAQD,QACjB,IAAIC,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,AAAD,MAAM,aAAaH;wBAClC;oBAEJ;oBACA,OAAOA;gBACT,EAAE,OAAM;oBACN;gBACF;YACF;YAEA,MAAMI,UAAU;gBACdf;gBACAS;gBACA,eAAeF,SAAS,MAAM;gBAC9BG;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoCR,cAAca;YAEnE,MAAMC,WAAWT,QAAQ,CAAC,EAAE;YAC5B,IAAIS,UACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0Cd,cAAcc,UAAU;QAExF,EAAE,OAAOC,GAAG;YACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2CA;QAC/D;IACF;IAEA,MAAM,qBAAqBC,SAAiB,EAAE;QAC5C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwBA;IAC3C;IAEA,MAAM,eAAelB,EAAU,EAAiB,CAEhD;IAEA,MAAM,iBACJA,EAAU,EACVmB,QAA8C,EAC9CC,IAAa,EACb;QACA,OAAOA;IACT;IAEA,MAAM,gBACJpB,EAAU,EACVmB,QAA8C,EAC9CtB,MAAe,EACG;QAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4BD,KAAK,SAAS,CAACuB;QAE5D,IAAIA,SAAS,IAAI,KAAK3B,uBAAuB,YAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAInB,MAAM6B,MAAM,IAAI,CAAC,gBAAgB;QAEjC,MAAMC,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,YAAY;QAChD,IAAI,CAACA,QAAQ,YACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAMC,aAAa,IAAIC,kBAAkBF,OAAO,MAAM;QACtD,MAAMG,YAAYF,WAAW,UAAU;QACvC,IAAI,CAACE,WAAW,YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAM,EAAE,OAAOC,UAAU,EAAE,QAAQC,WAAW,EAAE,GAAGJ,WAAW,aAAa,MAAM;YAC/E,OAAO;YACP,QAAQ;QACV;QAEA,MAAMV,UAAuC;YAC3C;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKY;oBACL,QAAQ,IAAI,CAAC,gBAAgB,CAACC,YAAYC;gBAC5C;YACF;YACA;gBACE,MAAM;gBACN,MAAM,CAAC,mBAAmB,EAAED,WAAW,CAAC,EAAEC,YAAY,qGAAqG,CAAC;YAC9J;SACD;QAED,IAAIL,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG,EACbT,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,CAAC,wBAAwB,EAAES,OAAO,GAAG,EAAE;QAC/C;QAGF,MAAMM,cAAc,IAAI,CAAC,cAAc;QACvC,MAAMC,SAASD,YAAY,SAAS;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkCC,OAAO,MAAM;QAEhE,MAAMC,QAAQF,YAAY,WAAW,CAAC,qBAAqB;YACzD,aAAa;YACbf;YACA,UAAU;gBACR,MAAM;gBACN,KAAKS,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG;YAClB;QACF;QACAM,YAAY,SAAS,CAACE;QACtB,OAAOjC;IACT;IA9NA,YAAYkC,MAAyB,CAAE;QACrC,MAAM,EACJC,QAAQ,EACRvB,KAAK,EACLwB,YAAY,EACZC,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,YAAY,EACZC,mBAAmB,GAAG,EACvB,GAAGP;QACJ,IAAIQ,oBAAoBC;QACxB,IAAI,AAAwB,YAAxB,OAAOP,cACTM,oBAAoBN;aACf,IAAIzB,MAAM,OAAO,CAACyB,eACvBM,oBAAoBN,aACjB,GAAG,CAAC,CAACQ,IAAO,AAAa,YAAb,OAAOA,IAAiBA,IAAIA,EAAE,OAAO,EACjD,IAAI,CAAC;aACH,IAAIR,gBAAgBS,uBAAuBT,eAChDM,oBAAoBI,qBAAqBV,cAAcD,SAAS,mBAAmB;QAErF9C,cAAc,KAAK,CAAC,8BAA8BqD;QAGlD,MAAMK,wBAAwB,cAAcC;YAC1C,aAAc;gBACZ,KAAK,CAACX;YACR;QACF;QAEA,KAAK,CAAC;YACJ,MAAM7C,SAAS,KAAK;YACpB,cAAckD;YACd,OAAO,EAAE;YACT,gBAAgBK;YAChB,OAAOnC;YACP,GAAI4B,gBAAgB;gBAAE,eAAeA;YAAa,CAAC;YACnD,UAAUjD,SAAS,KAAK;QAC1B,IA3CF,uBAAQ,YAAR,SACA,uBAAQ,wBAAR,SACA,uBAAQ,oBAAR,SACA,uBAAQ,oBAAR;QAyCE,IAAI,CAAC,QAAQ,GAAG4C;QAChB,IAAI,CAAC,oBAAoB,GAAGG,wBAAwBW;QAEpD,IAAI,CAAC,gBAAgB,GAAGV,oBAAoBW;QAC5C,IAAI,CAAC,gBAAgB,GAAGT;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAClC;AAkLF;AAtOE,iBADWjD,UACJ,SAAQ"}