{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToString } from '@/common';\nimport type { PlanningAIResponse, PlanningAction } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport { prepareModelImage } from '../image-preprocess';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendHighPriorityKnowledge(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return (\n    systemPrompt +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\\n`\n      : '')\n  );\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ChatCompletionMessageParam[] {\n  const { options, userInstructionText } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendHighPriorityKnowledge(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.append({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: { url: context.screenshot.base64 },\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...(options.referenceImageMessages ?? []),\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...(options.referenceImageMessages ?? []),\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: TUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    userInstructionText: userPromptToString(userInstruction),\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultAdapter: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  if (assistantContent) {\n    options.conversationHistory.append({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawChoiceMessage,\n  };\n}\n"],"names":["appendHighPriorityKnowledge","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","userInstructionText","conversationHistory","context","userInstruction","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","userPromptToString","messages","content","usage","rawChoiceMessage","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAiBA,SAASA,4BACPC,YAAoB,EACpBC,aAAsB;IAEtB,OACED,eACCC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,4BAA4B,CAAC,GACvE,EAAC;AAET;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGH;IACzC,MAAM,EAAEI,mBAAmB,EAAEC,OAAO,EAAEP,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,4BACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMQ,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACE,uBAC5BA;IAEJ,IAAIC,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBAAE,KAAKC,QAAQ,UAAU,CAAC,MAAM;gBAAC;YAC9C;SACD;IACH;IAEA,IAAIJ,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACIJ,QAAQ,sBAAsB,IAAI,EAAE;WACrCE,oBAAoB,QAAQ,CAACH,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACIJ,QAAQ,sBAAsB,IAAI,EAAE;WACrCE,oBAAoB,QAAQ,CAACH,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeM,kBACpBD,eAA4B,EAC5BJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEI,OAAO,EAAE,GAAGH;IACpB,MAAMM,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaJ,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQH,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMQ,kBAA+B;QACnC,GAAGR,OAAO;QACV,SAAS;YACP,GAAGG,OAAO;YACV,YAAYM,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBH,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUG,cAAc,YAAY;QACtC;IACF;IACA,MAAMR,QAA6B;QACjCM;QACA,qBAAqBM,mBAAmBN;QACxC,SAASI;QACT,kBAAkBT,OAAO,gBAAgB;IAC3C;IAEA,MAAMY,WAAWd,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEa,OAAO,EAAEC,KAAK,EAAEC,gBAAgB,EAAE,GAAG,MAAMC,yBACjDJ,UACAH,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIQ;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASjB,OAAO,aAAa,CAACa,SAASd;QACvCmB,UAAUlB,OAAO,gBAAgB,CAACiB,QAAQlB;QAC1CqB,oCAAoCF,SAAS;YAC3C,aAAaT,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,qBAAqBT,OAAO,oBAAoB;YAChD,qBAAqB;gBACnB,cAAcO,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;QACF;QACAY,yBAAyBnB,OAAO,sBAAsB,CAACiB,QAAQC;IACjE,EAAE,OAAOG,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACb,SAASc,QAAW,IACnCb,OACAC;IAEJ;IAEA,MAAMa,mBAAmB5B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DiB,QACAJ,SACAd;IAEF,IAAI6B,kBACF3B,QAAQ,mBAAmB,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,SAAS2B;IACX;IAGF,OAAO;QACLV;QACA,KAAKlB,OAAO,gBAAgB,CAACiB,QAAQJ;QACrCC;QACAK;QACA,aAAaO,KAAK,SAAS,CAACb,SAASc,QAAW;QAChDZ;IACF;AACF"}