{"version":3,"file":"ai-model/models/ui-tars/actions.mjs","sources":["../../../../../src/ai-model/models/ui-tars/actions.ts"],"sourcesContent":["import type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  DragAndDropPlanningAction,\n  LocatePlanningAction,\n} from '../../shared/planning-action';\nimport type { UiTarsParsedPlanningResponse } from './parser';\n\nconst debug = getDebug('ui-tars-planning');\nconst warnLog = getDebug('ui-tars-planning', { console: true });\n\nexport function transformUiTarsActions(\n  parsedPlanningResponse: UiTarsParsedPlanningResponse,\n): PlanningAction[] {\n  const transformActions: PlanningAction[] = [];\n  const unhandledActions: Array<{ type: string; thought: string }> = [];\n\n  parsedPlanningResponse.actions.forEach((action) => {\n    const actionType = (action.action_type || '').toLowerCase();\n    if (actionType === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate,\n        },\n      } satisfies LocatePlanningAction<'Tap'>);\n    } else if (actionType === 'left_double') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'DoubleClick',\n        param: {\n          locate,\n        },\n        thought: action.thought || '',\n      } satisfies LocatePlanningAction<'DoubleClick'>);\n    } else if (actionType === 'right_single') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'RightClick',\n        param: {\n          locate,\n        },\n        thought: action.thought || '',\n      } satisfies LocatePlanningAction<'RightClick'>);\n    } else if (actionType === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box);\n      const endPoint = getPoint(action.action_inputs.end_box);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            point: startPoint,\n            prompt: action.thought || '',\n          },\n          to: {\n            point: endPoint,\n            prompt: action.thought || '',\n          },\n        },\n        thought: action.thought || '',\n      } satisfies DragAndDropPlanningAction);\n    } else if (actionType === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'finished') {\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.action_inputs.content || action.thought || '',\n      });\n    } else if (actionType === 'hotkey') {\n      if (!action.action_inputs.key) {\n        warnLog('No key found in action: hotkey. Will not perform action.');\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys.join('+'),\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (actionType === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType) {\n      unhandledActions.push({\n        type: actionType,\n        thought: action.thought || '',\n      });\n      debug('Unhandled action type:', actionType, 'thought:', action.thought);\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(\n      buildNoUiTarsActionsError(\n        parsedPlanningResponse.rawResponse,\n        parsedPlanningResponse.actions,\n        unhandledActions,\n      ),\n    );\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n  return transformActions;\n}\n\nfunction getPoint(startBox: string): [number, number] {\n  const [x, y] = JSON.parse(startBox);\n  assert(\n    typeof x === 'number' &&\n      Number.isFinite(x) &&\n      typeof y === 'number' &&\n      Number.isFinite(y),\n    `invalid point data for ui-tars planning: ${startBox}`,\n  );\n  return [x, y];\n}\n\nfunction buildNoUiTarsActionsError(\n  rawResponse: string,\n  actions: UiTarsParsedPlanningResponse['actions'],\n  unhandledActions: Array<{ type: string; thought: string }>,\n): string {\n  const errorDetails: string[] = [];\n\n  if (actions.length === 0) {\n    errorDetails.push('Action parser returned no actions');\n\n    if (rawResponse.includes('Thought:') && !rawResponse.includes('Action:')) {\n      errorDetails.push(\n        'Response contains \"Thought:\" but missing \"Action:\" line',\n      );\n    } else {\n      errorDetails.push('Response may be malformed or empty');\n    }\n  }\n\n  if (unhandledActions.length > 0) {\n    const types = unhandledActions.map((a) => a.type).join(', ');\n    errorDetails.push(`Unhandled action types: ${types}`);\n  }\n\n  return ['No actions found in UI-TARS response.', ...errorDetails].join('\\n');\n}\n"],"names":["debug","getDebug","warnLog","transformUiTarsActions","parsedPlanningResponse","transformActions","unhandledActions","action","actionType","assert","point","getPoint","locate","startPoint","endPoint","keys","transformHotkeyInput","Error","buildNoUiTarsActionsError","JSON","startBox","x","y","Number","rawResponse","actions","errorDetails","types","a"],"mappings":";;;AAUA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,oBAAoB;IAAE,SAAS;AAAK;AAEtD,SAASE,uBACdC,sBAAoD;IAEpD,MAAMC,mBAAqC,EAAE;IAC7C,MAAMC,mBAA6D,EAAE;IAErEF,uBAAuB,OAAO,CAAC,OAAO,CAAC,CAACG;QACtC,MAAMC,aAAcD,AAAAA,CAAAA,OAAO,WAAW,IAAI,EAAC,EAAG,WAAW;QACzD,IAAIC,AAAe,YAAfA,YAAwB;YAC1BC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;YACF;QACF,OAAO,IAAIJ,AAAe,kBAAfA,YAA8B;YACvCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,mBAAfA,YAA+B;YACxCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YAAuB;YAChCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCE,OAAOF,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMM,aAAaF,SAASJ,OAAO,aAAa,CAAC,SAAS;YAC1D,MAAMO,WAAWH,SAASJ,OAAO,aAAa,CAAC,OAAO;YACtDF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,OAAOQ;wBACP,QAAQN,OAAO,OAAO,IAAI;oBAC5B;oBACA,IAAI;wBACF,OAAOO;wBACP,QAAQP,OAAO,OAAO,IAAI;oBAC5B;gBACF;gBACA,SAASA,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOE,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,aAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWE,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,eAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO,CAAC;YACR,SAASE,OAAO,aAAa,CAAC,OAAO,IAAIA,OAAO,OAAO,IAAI;QAC7D;aACK,IAAIC,AAAe,aAAfA,YACT,IAAKD,OAAO,aAAa,CAAC,GAAG,EAEtB;YACL,MAAMQ,OAAOC,qBAAqBT,OAAO,aAAa,CAAC,GAAG;YAE1DF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASU,KAAK,IAAI,CAAC;gBACrB;gBACA,SAASR,OAAO,OAAO,IAAI;YAC7B;QACF,OAXEL,QAAQ;aAYL,IAAIM,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASE,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,YAAY;YACrBF,iBAAiB,IAAI,CAAC;gBACpB,MAAME;gBACN,SAASD,OAAO,OAAO,IAAI;YAC7B;YACAP,MAAM,0BAA0BQ,YAAY,YAAYD,OAAO,OAAO;QACxE;IACF;IAEA,IAAIF,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIY,MACRC,0BACEd,uBAAuB,WAAW,EAClCA,uBAAuB,OAAO,EAC9BE;IAKNN,MAAM,oBAAoBmB,KAAK,SAAS,CAACd,kBAAkB,MAAM;IACjE,OAAOA;AACT;AAEA,SAASM,SAASS,QAAgB;IAChC,MAAM,CAACC,GAAGC,EAAE,GAAGH,KAAK,KAAK,CAACC;IAC1BX,OACE,AAAa,YAAb,OAAOY,KACLE,OAAO,QAAQ,CAACF,MAChB,AAAa,YAAb,OAAOC,KACPC,OAAO,QAAQ,CAACD,IAClB,CAAC,yCAAyC,EAAEF,UAAU;IAExD,OAAO;QAACC;QAAGC;KAAE;AACf;AAEA,SAASJ,0BACPM,WAAmB,EACnBC,OAAgD,EAChDnB,gBAA0D;IAE1D,MAAMoB,eAAyB,EAAE;IAEjC,IAAID,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;QACxBC,aAAa,IAAI,CAAC;QAElB,IAAIF,YAAY,QAAQ,CAAC,eAAe,CAACA,YAAY,QAAQ,CAAC,YAC5DE,aAAa,IAAI,CACf;aAGFA,aAAa,IAAI,CAAC;IAEtB;IAEA,IAAIpB,iBAAiB,MAAM,GAAG,GAAG;QAC/B,MAAMqB,QAAQrB,iBAAiB,GAAG,CAAC,CAACsB,IAAMA,EAAE,IAAI,EAAE,IAAI,CAAC;QACvDF,aAAa,IAAI,CAAC,CAAC,wBAAwB,EAAEC,OAAO;IACtD;IAEA,OAAO;QAAC;WAA4CD;KAAa,CAAC,IAAI,CAAC;AACzE"}