{"version":3,"file":"ai-model/connectivity/run-connectivity-test.mjs","sources":["../../../../src/ai-model/connectivity/run-connectivity-test.ts"],"sourcesContent":["import { ScreenshotItem } from '@/screenshot-item';\nimport Service from '@/service';\nimport type { UIContext } from '@/types';\nimport type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getModelRuntime } from '../models';\nimport type { ModelRuntime } from '../models';\nimport { callAI } from '../service-caller';\nimport {\n  CONNECTIVITY_FIXTURE_IMAGE,\n  CONNECTIVITY_FIXTURE_SHOT_SIZE,\n} from './fixture';\n\nconst TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';\n\ninterface ConnectivityCheckResultItem {\n  name: 'text' | 'vision' | 'aiLocate';\n  intent: TIntent;\n  modelName: string;\n  modelFamily?: string;\n  passed: boolean;\n  durationMs: number;\n  message: string;\n}\n\nexport interface ConnectivityTestResult {\n  passed: boolean;\n  message?: string;\n}\n\nexport interface ConnectivityTestConfig {\n  defaultModelConfig: IModelConfig;\n  planningModelConfig: IModelConfig;\n  insightModelConfig: IModelConfig;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction hasValidRect(value: unknown): boolean {\n  if (!value || typeof value !== 'object') {\n    return false;\n  }\n\n  const rect = value as {\n    left?: unknown;\n    top?: unknown;\n    width?: unknown;\n    height?: unknown;\n  };\n\n  return (\n    isFiniteNumber(rect.left) &&\n    isFiniteNumber(rect.top) &&\n    isFiniteNumber(rect.width) &&\n    isFiniteNumber(rect.height)\n  );\n}\n\nfunction hasValidCenter(value: unknown): boolean {\n  return (\n    Array.isArray(value) &&\n    value.length === 2 &&\n    isFiniteNumber(value[0]) &&\n    isFiniteNumber(value[1])\n  );\n}\n\nfunction buildFixtureContext(): UIContext {\n  return {\n    screenshot: ScreenshotItem.create(CONNECTIVITY_FIXTURE_IMAGE, Date.now()),\n    shotSize: CONNECTIVITY_FIXTURE_SHOT_SIZE,\n    shrunkShotToLogicalRatio: 1,\n  };\n}\n\nfunction buildCheckResult(\n  name: ConnectivityCheckResultItem['name'],\n  modelRuntime: ModelRuntime,\n  result: Omit<\n    ConnectivityCheckResultItem,\n    'name' | 'intent' | 'modelName' | 'modelFamily'\n  >,\n): ConnectivityCheckResultItem {\n  const { config } = modelRuntime;\n  return {\n    name,\n    intent: config.intent,\n    modelName: config.modelName,\n    modelFamily: config.modelFamily,\n    ...result,\n  };\n}\n\nfunction buildConnectivityModelRuntime(\n  modelConfig: IModelConfig,\n): ModelRuntime {\n  return getModelRuntime({\n    ...modelConfig,\n    retryCount: 0,\n  });\n}\n\nfunction formatConnectivityCheckName(\n  check: ConnectivityCheckResultItem,\n): string {\n  const checkName =\n    check.name === 'aiLocate'\n      ? 'AI locate check'\n      : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;\n  const modelLabel = check.modelName || check.intent;\n  return `${checkName} - ${modelLabel} (${check.intent})`;\n}\n\nfunction buildConnectivityFailureMessage(\n  checks: ConnectivityCheckResultItem[],\n): string {\n  const failedChecks = checks.filter((item) => !item.passed);\n  if (failedChecks.length === 0) {\n    return 'Connectivity test failed, but no failed check details were generated.';\n  }\n\n  return failedChecks\n    .map((item) => {\n      const detail = item.message || 'Failed without details.';\n      return `[${formatConnectivityCheckName(item)}]: ${detail}`;\n    })\n    .join('\\n');\n}\n\nasync function runTextConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const result = await callAI(\n      [\n        {\n          role: 'system',\n          content: 'Reply with the exact token the user asks for.',\n        },\n        {\n          role: 'user',\n          content: `Return exactly ${TEXT_EXPECTED_TOKEN}`,\n        },\n      ],\n      modelRuntime,\n    );\n    const content = result.content.trim();\n    const passed = content.includes(TEXT_EXPECTED_TOKEN);\n    return buildCheckResult('text', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed ? '' : `Unexpected response: ${content}`,\n    });\n  } catch (error) {\n    return buildCheckResult('text', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runVisionConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    await callAI(\n      [\n        {\n          role: 'user',\n          content: [\n            {\n              type: 'text',\n              text: 'What is the main content of this image ? It is a photo or a form ?',\n            },\n            {\n              type: 'image_url',\n              image_url: {\n                url: CONNECTIVITY_FIXTURE_IMAGE,\n                detail: 'high',\n              },\n            },\n          ],\n        },\n      ],\n      modelRuntime,\n    );\n    return buildCheckResult('vision', modelRuntime, {\n      passed: true,\n      durationMs: Date.now() - startTime,\n      message: '',\n    });\n  } catch (error) {\n    return buildCheckResult('vision', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runAiLocateConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const context = buildFixtureContext();\n    const service = new Service(context);\n    const locateResult = await service.locate(\n      { prompt: 'the main todo input box' },\n      {},\n      modelRuntime,\n    );\n    const targetRect = locateResult.rect || locateResult.element?.rect;\n    const center = locateResult.element?.center;\n    const passed = hasValidRect(targetRect) && hasValidCenter(center);\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed\n        ? ''\n        : `Invalid locate result: ${JSON.stringify({\n            rect: targetRect,\n            center,\n          })}`,\n    });\n  } catch (error) {\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nexport async function runConnectivityTest(\n  config: ConnectivityTestConfig,\n): Promise<ConnectivityTestResult> {\n  const planningModelRuntime = buildConnectivityModelRuntime(\n    config.planningModelConfig,\n  );\n  const insightModelRuntime = buildConnectivityModelRuntime(\n    config.insightModelConfig,\n  );\n  const defaultModelRuntime = buildConnectivityModelRuntime(\n    config.defaultModelConfig,\n  );\n  const checks = await Promise.all([\n    runTextConnectivityCheck(planningModelRuntime),\n    runVisionConnectivityCheck(insightModelRuntime),\n    runAiLocateConnectivityCheck(defaultModelRuntime),\n  ]);\n\n  const passed = checks.every((item) => item.passed);\n  return {\n    passed,\n    message: passed ? undefined : buildConnectivityFailureMessage(checks),\n  };\n}\n"],"names":["TEXT_EXPECTED_TOKEN","isFiniteNumber","value","Number","hasValidRect","rect","hasValidCenter","Array","buildFixtureContext","ScreenshotItem","CONNECTIVITY_FIXTURE_IMAGE","Date","CONNECTIVITY_FIXTURE_SHOT_SIZE","buildCheckResult","name","modelRuntime","result","config","buildConnectivityModelRuntime","modelConfig","getModelRuntime","formatConnectivityCheckName","check","checkName","modelLabel","buildConnectivityFailureMessage","checks","failedChecks","item","detail","runTextConnectivityCheck","startTime","callAI","content","passed","error","Error","String","runVisionConnectivityCheck","runAiLocateConnectivityCheck","context","service","Service","locateResult","targetRect","center","JSON","runConnectivityTest","planningModelRuntime","insightModelRuntime","defaultModelRuntime","Promise","undefined"],"mappings":";;;;;AAYA,MAAMA,sBAAsB;AAuB5B,SAASC,eAAeC,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AACtD;AAEA,SAASE,aAAaF,KAAc;IAClC,IAAI,CAACA,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAMG,OAAOH;IAOb,OACED,eAAeI,KAAK,IAAI,KACxBJ,eAAeI,KAAK,GAAG,KACvBJ,eAAeI,KAAK,KAAK,KACzBJ,eAAeI,KAAK,MAAM;AAE9B;AAEA,SAASC,eAAeJ,KAAc;IACpC,OACEK,MAAM,OAAO,CAACL,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZD,eAAeC,KAAK,CAAC,EAAE,KACvBD,eAAeC,KAAK,CAAC,EAAE;AAE3B;AAEA,SAASM;IACP,OAAO;QACL,YAAYC,eAAe,MAAM,CAACC,4BAA4BC,KAAK,GAAG;QACtE,UAAUC;QACV,0BAA0B;IAC5B;AACF;AAEA,SAASC,iBACPC,IAAyC,EACzCC,YAA0B,EAC1BC,MAGC;IAED,MAAM,EAAEC,MAAM,EAAE,GAAGF;IACnB,OAAO;QACLD;QACA,QAAQG,OAAO,MAAM;QACrB,WAAWA,OAAO,SAAS;QAC3B,aAAaA,OAAO,WAAW;QAC/B,GAAGD,MAAM;IACX;AACF;AAEA,SAASE,8BACPC,WAAyB;IAEzB,OAAOC,gBAAgB;QACrB,GAAGD,WAAW;QACd,YAAY;IACd;AACF;AAEA,SAASE,4BACPC,KAAkC;IAElC,MAAMC,YACJD,AAAe,eAAfA,MAAM,IAAI,GACN,oBACA,GAAGA,MAAM,IAAI,CAAC,EAAE,EAAE,gBAAgBA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACnE,MAAME,aAAaF,MAAM,SAAS,IAAIA,MAAM,MAAM;IAClD,OAAO,GAAGC,UAAU,GAAG,EAAEC,WAAW,EAAE,EAAEF,MAAM,MAAM,CAAC,CAAC,CAAC;AACzD;AAEA,SAASG,gCACPC,MAAqC;IAErC,MAAMC,eAAeD,OAAO,MAAM,CAAC,CAACE,OAAS,CAACA,KAAK,MAAM;IACzD,IAAID,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAO;IAGT,OAAOA,aACJ,GAAG,CAAC,CAACC;QACJ,MAAMC,SAASD,KAAK,OAAO,IAAI;QAC/B,OAAO,CAAC,CAAC,EAAEP,4BAA4BO,MAAM,GAAG,EAAEC,QAAQ;IAC5D,GACC,IAAI,CAAC;AACV;AAEA,eAAeC,yBACbf,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMK,SAAS,MAAMgB,OACnB;YACE;gBACE,MAAM;gBACN,SAAS;YACX;YACA;gBACE,MAAM;gBACN,SAAS,CAAC,eAAe,EAAEhC,qBAAqB;YAClD;SACD,EACDe;QAEF,MAAMkB,UAAUjB,OAAO,OAAO,CAAC,IAAI;QACnC,MAAMkB,SAASD,QAAQ,QAAQ,CAACjC;QAChC,OAAOa,iBAAiB,QAAQE,cAAc;YAC5CmB;YACA,YAAYvB,KAAK,GAAG,KAAKoB;YACzB,SAASG,SAAS,KAAK,CAAC,qBAAqB,EAAED,SAAS;QAC1D;IACF,EAAE,OAAOE,OAAO;QACd,OAAOtB,iBAAiB,QAAQE,cAAc;YAC5C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeG,2BACbvB,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMqB,OACJ;YACE;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM;oBACR;oBACA;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKtB;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD,EACDK;QAEF,OAAOF,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAAS;QACX;IACF,EAAE,OAAOI,OAAO;QACd,OAAOtB,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeI,6BACbxB,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAM6B,UAAUhC;QAChB,MAAMiC,UAAU,IAAIC,UAAQF;QAC5B,MAAMG,eAAe,MAAMF,QAAQ,MAAM,CACvC;YAAE,QAAQ;QAA0B,GACpC,CAAC,GACD1B;QAEF,MAAM6B,aAAaD,aAAa,IAAI,IAAIA,aAAa,OAAO,EAAE;QAC9D,MAAME,SAASF,aAAa,OAAO,EAAE;QACrC,MAAMT,SAAS9B,aAAawC,eAAetC,eAAeuC;QAC1D,OAAOhC,iBAAiB,YAAYE,cAAc;YAChDmB;YACA,YAAYvB,KAAK,GAAG,KAAKoB;YACzB,SAASG,SACL,KACA,CAAC,uBAAuB,EAAEY,KAAK,SAAS,CAAC;gBACvC,MAAMF;gBACNC;YACF,IAAI;QACV;IACF,EAAE,OAAOV,OAAO;QACd,OAAOtB,iBAAiB,YAAYE,cAAc;YAChD,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEO,eAAeY,oBACpB9B,MAA8B;IAE9B,MAAM+B,uBAAuB9B,8BAC3BD,OAAO,mBAAmB;IAE5B,MAAMgC,sBAAsB/B,8BAC1BD,OAAO,kBAAkB;IAE3B,MAAMiC,sBAAsBhC,8BAC1BD,OAAO,kBAAkB;IAE3B,MAAMS,SAAS,MAAMyB,QAAQ,GAAG,CAAC;QAC/BrB,yBAAyBkB;QACzBV,2BAA2BW;QAC3BV,6BAA6BW;KAC9B;IAED,MAAMhB,SAASR,OAAO,KAAK,CAAC,CAACE,OAASA,KAAK,MAAM;IACjD,OAAO;QACLM;QACA,SAASA,SAASkB,SAAY3B,gCAAgCC;IAChE;AACF"}