""" skills/gui_skill.py - 桌面 GUI 自动化技能 ========================================= 提供跨平台桌面 GUI 自动化能力，包括屏幕截图、鼠标操作、键盘输入、窗口管理等。 支持 Windows 和 macOS。所有依赖均为可选，缺失时给出友好的安装提示。 Skills: - ScreenShotSkill: 捕获屏幕或区域截图 - MouseClickSkill: 在屏幕坐标处点击 - MouseDragSkill: 从 A 点拖拽到 B 点 - TypeTextSkill: 在当前光标位置输入文本 - HotkeySkill: 按下键盘快捷键（自动适配平台） - WindowListSkill: 列出所有打开的窗口 - WindowFocusSkill: 聚焦/置顶指定窗口 - ScreenElementSkill: 使用 VLM 从截图中定位元素坐标 Dependencies (all optional): - mss: 屏幕截图 (跨平台, ~1MB) - pynput: 鼠标/键盘控制 (跨平台, ~500KB) - pygetwindow: 窗口管理 (Windows + macOS) """ from __future__ import annotations import sys import time from typing import Any, Dict, List, Optional, Tuple from core.logger import get_logger from aiskills.base import Skill, SkillResult, SkillParameter logger = get_logger("myagent.skills.gui") # 平台检测 / Platform detection IS_MACOS = sys.platform == "darwin" IS_WINDOWS = sys.platform == "win32" IS_LINUX = sys.platform.startswith("linux") def _generate_screenshot_path() -> str: """生成截图文件路径（带时间戳）""" timestamp = time.strftime("%Y%m%d_%H%M%S") return f"/tmp/myagent_gui_screen_{timestamp}.png" class ScreenShotSkill(Skill): """ 屏幕截图 - 捕获整个屏幕或指定区域的截图。 使用 mss 库实现跨平台高速截图，比 Pillow.grab 快 10 倍以上。 截图保存为 PNG 文件，返回路径供 VLM 分析。 """ name = "screenshot" description = ( "捕获屏幕截图，保存为 PNG 文件并返回路径。" "可截取全屏或指定区域。截图可用于 VLM 视觉分析以定位 UI 元素。" ) category = "gui" parameters = [ SkillParameter("region", "string", "截图区域坐标 'left,top,width,height'（留空则截取全屏）。" "例如: '100,200,800,600' 表示从 (100,200) 开始截取 800x600 区域", required=False, default=""), SkillParameter("monitor", "integer", "显示器编号（多显示器环境），0=全部, 1=主显示器, 2=副显示器...", required=False, default=1), ] async def execute( self, region: str = "", monitor: int = 1, **kwargs, ) -> SkillResult: """执行：捕获屏幕截图""" try: import mss except ImportError: from core.deps_checker import ensure_skill_deps if not ensure_skill_deps("gui"): return SkillResult( success=False, error="mss 安装失败，请手动运行: pip install mss", ) import mss try: screenshot_path = _generate_screenshot_path() with mss.mss() as sct: # 确定截图区域 if region: # 解析区域: "left,top,width,height" parts = region.split(",") if len(parts) != 4: return SkillResult( success=False, error=f"区域格式错误，应为 'left,top,width,height'，得到: {region}", ) try: left, top, width, height = [int(p.strip()) for p in parts] except ValueError: return SkillResult(success=False, error=f"区域坐标必须是整数: {region}") monitor_info = {"left": left, "top": top, "width": width, "height": height} else: # 截取指定显示器（默认主显示器） monitors = sct.monitors if monitor < 0 or monitor >= len(monitors): return SkillResult( success=False, error=f"显示器编号 {monitor} 无效，可用范围: 0-{len(monitors) - 1}", ) monitor_info = monitors[monitor] # 捕获截图 sct_img = sct.grab(monitor_info) # 保存为 PNG from mss.tools import to_png mss.tools.to_png(sct_img.rgb, sct_img.size, output=screenshot_path) import os file_size = os.path.getsize(screenshot_path) return SkillResult( success=True, data={ "screenshot_path": screenshot_path, "file_size_bytes": file_size, "width": monitor_info.get("width"), "height": monitor_info.get("height"), "region": region or None, "monitor": monitor, }, message=f"屏幕截图已保存: {screenshot_path} ({file_size} 字节, {monitor_info.get('width')}x{monitor_info.get('height')})", files=[screenshot_path], ) except Exception as e: logger.error(f"屏幕截图失败: {e}") return SkillResult(success=False, error=f"屏幕截图失败: {e}") class MouseClickSkill(Skill): """ 鼠标点击 - 在指定的屏幕坐标处执行鼠标点击。 支持单击、双击、右键点击。坐标以屏幕左上角为原点 (0, 0)。 需要先用 screenshot 技能截图，然后用 VLM 分析获取坐标。 """ name = "mouse_click" description = ( "在指定屏幕坐标处执行鼠标点击操作。" "支持左键单击、左键双击、右键单击。" "坐标以屏幕左上角为原点 (0,0)。" ) category = "gui" dangerous = True parameters = [ SkillParameter("x", "integer", "目标 X 坐标（屏幕水平位置，左上角为 0）", required=True), SkillParameter("y", "integer", "目标 Y 坐标（屏幕垂直位置，左上角为 0）", required=True), SkillParameter("button", "string", "鼠标按钮: left=左键, right=右键, middle=中键", required=False, default="left", enum=["left", "right", "middle"]), SkillParameter("clicks", "integer", "点击次数: 1=单击, 2=双击, 3=三击", required=False, default=1), ] async def execute( self, x: int = 0, y: int = 0, button: str = "left", clicks: int = 1, **kwargs, ) -> SkillResult: """执行：鼠标点击""" try: from pynput.mouse import Controller, Button except Exception: return SkillResult( success=False, error="pynput 不可用（当前平台缺少 GUI 环境或未安装）。仅在 Windows/macOS 桌面环境支持鼠标操作。", ) try: mouse = Controller() button_map = { "left": Button.left, "right": Button.right, "middle": Button.middle, } btn = button_map.get(button, Button.left) # 移动到目标位置 mouse.position = (x, y) time.sleep(0.05) # 短暂等待鼠标移动到位 # 执行点击 if clicks == 1: mouse.click(btn, 1) elif clicks == 2: mouse.click(btn, 2) else: mouse.click(btn, clicks) click_desc = {"left": "左键", "right": "右键", "middle": "中键"}.get(button, button) count_desc = "单击" if clicks == 1 else f"{clicks}连击" return SkillResult( success=True, data={"x": x, "y": y, "button": button, "clicks": clicks}, message=f"已在 ({x}, {y}) 执行{click_desc}{count_desc}", ) except Exception as e: logger.error(f"鼠标点击失败: {e}") return SkillResult(success=False, error=f"鼠标点击失败: {e}") class MouseDragSkill(Skill): """ 鼠标拖拽 - 从起点拖拽到终点。 常用于文件拖放、滑动操作、选择区域等场景。 可设置拖拽持续时间和步数，实现平滑拖拽效果。 """ name = "mouse_drag" description = ( "从起点坐标拖拽到终点坐标。可用于文件拖放、区域选择、滑动条操作等。" "支持设置拖拽持续时间以实现平滑拖拽效果。" ) category = "gui" dangerous = True parameters = [ SkillParameter("start_x", "integer", "起点 X 坐标", required=True), SkillParameter("start_y", "integer", "起点 Y 坐标", required=True), SkillParameter("end_x", "integer", "终点 X 坐标", required=True), SkillParameter("end_y", "integer", "终点 Y 坐标", required=True), SkillParameter("button", "string", "鼠标按钮: left=左键(默认), right=右键", required=False, default="left", enum=["left", "right"]), SkillParameter("duration", "float", "拖拽持续时间（秒），默认 0.3 秒", required=False, default=0.3), ] async def execute( self, start_x: int = 0, start_y: int = 0, end_x: int = 0, end_y: int = 0, button: str = "left", duration: float = 0.3, **kwargs, ) -> SkillResult: """执行：鼠标拖拽""" try: from pynput.mouse import Controller, Button except Exception: return SkillResult( success=False, error="pynput 不可用（当前平台缺少 GUI 环境或未安装）", ) try: mouse = Controller() button_map = { "left": Button.left, "right": Button.right, } btn = button_map.get(button, Button.left) # 移动到起点 mouse.position = (start_x, start_y) time.sleep(0.05) # 按下鼠标 mouse.press(btn) # 平滑移动到终点 steps = max(int(duration * 60), 1) # 大约 60fps dx = (end_x - start_x) / steps dy = (end_y - start_y) / steps step_delay = duration / steps for i in range(steps): mouse.move(int(dx), int(dy)) time.sleep(step_delay) # 确保到达精确位置 mouse.position = (end_x, end_y) # 释放鼠标 mouse.release(btn) return SkillResult( success=True, data={ "start": {"x": start_x, "y": start_y}, "end": {"x": end_x, "y": end_y}, "button": button, "duration": duration, }, message=f"已从 ({start_x}, {start_y}) 拖拽到 ({end_x}, {end_y})", ) except Exception as e: logger.error(f"鼠标拖拽失败: {e}") return SkillResult(success=False, error=f"鼠标拖拽失败: {e}") class TypeTextSkill(Skill): """ 输入文本 - 在当前光标位置输入文本。 使用 pynput 键盘控制器模拟真实按键输入。 支持普通文本和特殊按键。 输入前确保目标输入框已获取焦点。 """ name = "type_text" description = ( "在当前光标位置输入文本。模拟真实键盘按键输入。" "输入前请确保目标输入框已获取焦点（可先用 mouse_click 点击输入框）。" ) category = "gui" dangerous = True parameters = [ SkillParameter("text", "string", "要输入的文本内容", required=True), SkillParameter("interval", "float", "按键间隔（秒），默认 0.02 秒。增大可模拟更慢的打字", required=False, default=0.02), SkillParameter("clear_first", "boolean", "是否先用 Ctrl+A 全选再删除（清空已有内容）", required=False, default=False), ] async def execute( self, text: str = "", interval: float = 0.02, clear_first: bool = False, **kwargs, ) -> SkillResult: """执行：输入文本。优先使用 stealth 浏览器，否则回退 pynput。""" if not text: return SkillResult(success=False, error="缺少必需参数: text") # ── 优先：stealth 浏览器（适用于 web 场景） ── stealth_result = self._try_stealth_type(text, clear_first) if stealth_result is not None: return stealth_result # ── 回退：pynput 桌面键盘控制 ── try: from pynput.keyboard import Controller, Key except Exception: return SkillResult( success=False, error="无可用的输入方式（stealth 浏览器未启动，pynput 不可用）", ) try: keyboard = Controller() if clear_first: try: if IS_MACOS: keyboard.press(Key.cmd) else: keyboard.press(Key.ctrl) keyboard.press('a') keyboard.release('a') if IS_MACOS: keyboard.release(Key.cmd) else: keyboard.release(Key.ctrl) time.sleep(0.05) keyboard.press(Key.backspace) keyboard.release(Key.backspace) time.sleep(0.05) except Exception: pass keyboard.type(text) return SkillResult( success=True, data={"text_length": len(text), "cleared_first": clear_first}, message=f"已输入文本（{len(text)} 个字符）{f'（已先清空）' if clear_first else ''}", ) except Exception as e: logger.error(f"输入文本失败: {e}") return SkillResult(success=False, error=f"输入文本失败: {e}") @staticmethod def _try_stealth_type(text: str, clear_first: bool) -> "SkillResult | None": """尝试通过 stealth 浏览器输入文本，返回 None 表示 stealth 不可用。""" try: from aiskills.browser_stealth import get_stealth_browser, _browsers if not _browsers: return None profile = list(_browsers.keys())[-1] browser = get_stealth_browser(profile_name=profile) if not browser._ensure_page(): return None import asyncio loop = asyncio.get_event_loop() if loop.is_running(): return None return loop.run_until_complete( browser.type_text(text=text, clear=clear_first, delay_ms=0) ) except Exception: return None class HotkeySkill(Skill): """ 快捷键 - 按下键盘快捷键组合。 自动根据平台适配修饰键： - Windows/Linux: Ctrl 键 - macOS: Cmd (⌘) 键 支持常用快捷键如 Ctrl+C (复制), Ctrl+V (粘贴), Alt+Tab 等。 也可直接指定精确的按键组合。 """ name = "hotkey" description = ( "按下键盘快捷键。自动适配平台修饰键（Windows 用 Ctrl，Mac 用 Cmd）。" "支持预设快捷键（如 'copy', 'paste', 'select_all'）或自定义按键组合（如 'ctrl+c', 'alt+tab'）。" ) category = "gui" dangerous = True parameters = [ SkillParameter("action", "string", "快捷键动作或按键组合。预设值: copy, paste, cut, select_all, undo, redo, " "save, close, tab, new_tab, find, refresh, fullscreen, " "screenshot, quit, lock_screen。也可直接指定按键组合如 'ctrl+shift+i', 'alt+f4'", required=True), ] # 预设快捷键映射 / Preset hotkey mappings PRESET_HOTKEYS = { "copy": ["ctrl", "c"], "paste": ["ctrl", "v"], "cut": ["ctrl", "x"], "select_all": ["ctrl", "a"], "undo": ["ctrl", "z"], "redo": ["ctrl", "y"], "save": ["ctrl", "s"], "close": ["ctrl", "w"], "tab": ["ctrl", "tab"], "new_tab": ["ctrl", "t"], "find": ["ctrl", "f"], "refresh": ["ctrl", "r"], "fullscreen": ["ctrl", "f11"] if IS_WINDOWS else ["ctrl", "cmd", "f"], "screenshot": ["cmd", "shift", "3"] if IS_MACOS else ["ctrl", "print_screen"], "quit": ["cmd", "q"] if IS_MACOS else ["alt", "f4"], "lock_screen": ["ctrl", "cmd", "q"] if IS_MACOS else ["ctrl", "alt", "delete"], } async def execute(self, action: str = "", **kwargs) -> SkillResult: """执行：按下快捷键。优先使用 stealth 浏览器（如有活跃实例），否则回退 pynput。""" if not action: return SkillResult(success=False, error="缺少必需参数: action") # ── 优先：stealth 浏览器（适用于 web 场景） ── stealth_result = self._try_stealth_key(action) if stealth_result is not None: return stealth_result # ── 回退：pynput 桌面键盘控制 ── try: from pynput.keyboard import Controller, Key, KeyCode except Exception: return SkillResult( success=False, error="无可用的按键方式（stealth 浏览器未启动，pynput 不可用）", ) try: keyboard = Controller() if action.lower() in self.PRESET_HOTKEYS: keys = self.PRESET_HOTKEYS[action.lower()] else: keys = [k.strip().lower() for k in action.split("+")] pressed_keys = [] # 安全构建 key_map，只包含实际存在的属性 key_map = {} for attr_name in ('ctrl', 'alt', 'shift', 'cmd', 'tab', 'enter', 'space', 'backspace', 'delete', 'esc', 'up', 'down', 'left', 'right', 'home', 'end', 'page_up', 'page_down'): if hasattr(Key, attr_name): key_map[attr_name] = getattr(Key, attr_name) for i in range(1, 13): attr_name = f'f{i}' if hasattr(Key, attr_name): key_map[attr_name] = getattr(Key, attr_name) for attr_name in ('caps_lock', 'num_lock', 'insert', 'pause', 'print_screen'): if hasattr(Key, attr_name): key_map[attr_name] = getattr(Key, attr_name) if IS_MACOS: keys = ["cmd" if k == "ctrl" else k for k in keys] for key_name in keys: if key_name in key_map: keyboard.press(key_map[key_name]) pressed_keys.append(key_map[key_name]) elif len(key_name) == 1: keyboard.press(key_name) pressed_keys.append(key_name) time.sleep(0.05) for key in reversed(pressed_keys): keyboard.release(key) key_display = "+".join(keys) platform_note = " (macOS: Ctrl→Cmd)" if IS_MACOS else "" return SkillResult( success=True, data={"action": action, "keys": keys, "platform_adapted": IS_MACOS}, message=f"已按下快捷键: {key_display}{platform_note}", ) except Exception as e: logger.error(f"快捷键操作失败: {e}") return SkillResult(success=False, error=f"快捷键操作失败: {e}") @staticmethod def _try_stealth_key(action: str) -> "SkillResult | None": """尝试通过 stealth 浏览器按键，返回 None 表示 stealth 不可用。""" try: from aiskills.browser_stealth import get_stealth_browser, _browsers if not _browsers: return None profile = list(_browsers.keys())[-1] browser = get_stealth_browser(profile_name=profile) if not browser._ensure_page(): return None import asyncio loop = asyncio.get_event_loop() if loop.is_running(): return None return loop.run_until_complete(browser.press_key(key=action)) except Exception: return None class WindowListSkill(Skill): """ 窗口列表 - 列出当前所有打开的窗口。 返回每个窗口的标题、位置、大小等信息。 可用于定位目标窗口以便后续操作。 """ name = "window_list" description = ( "列出当前所有打开的窗口，包括标题、位置、大小等信息。" "可用于查找目标窗口标题，供 window_focus 使用。" ) category = "gui" parameters = [ SkillParameter("filter", "string", "窗口标题过滤关键词（留空则返回所有窗口）。仅返回标题包含该关键词的窗口", required=False, default=""), ] async def execute(self, filter: str = "", **kwargs) -> SkillResult: """执行：列出窗口""" try: import pygetwindow as gw except ImportError: return SkillResult(success=False, error="pygetwindow 未安装，请运行: pip install pygetwindow") except NotImplementedError: return SkillResult(success=False, error="当前平台不支持窗口管理（pygetwindow 仅支持 Windows 和 macOS）") try: windows = gw.getAllWindows() # 过滤窗口 if filter: windows = [w for w in windows if filter.lower() in w.title.lower()] window_list = [] for w in windows: window_list.append({ "title": w.title, "left": w.left, "top": w.top, "width": w.width, "height": w.height, "visible": w.visible, "active": w.isActive, }) return SkillResult( success=True, data={ "windows": window_list, "count": len(window_list), "filter": filter or None, "platform": sys.platform, }, message=f"共 {len(window_list)} 个窗口{f'（过滤: {filter}）' if filter else ''}", ) except Exception as e: logger.error(f"列出窗口失败: {e}") return SkillResult(success=False, error=f"列出窗口失败: {e}") class WindowFocusSkill(Skill): """ 窗口聚焦 - 将指定窗口置顶并获取焦点。 支持通过窗口标题（模糊匹配）或窗口标题序号来定位目标窗口。 聚焦后可进行后续的鼠标/键盘操作。 """ name = "window_focus" description = ( "将指定窗口置顶并获取焦点。支持通过窗口标题（模糊匹配）定位。" "聚焦后可在该窗口内进行鼠标点击、键盘输入等操作。" ) category = "gui" dangerous = True parameters = [ SkillParameter("title", "string", "目标窗口标题（模糊匹配，包含该文本即匹配）", required=True), SkillParameter("activate", "boolean", "是否激活窗口（默认 true）", required=False, default=True), SkillParameter("maximize", "boolean", "是否最大化窗口（默认 false）", required=False, default=False), ] async def execute( self, title: str = "", activate: bool = True, maximize: bool = False, **kwargs, ) -> SkillResult: """执行：聚焦窗口""" try: import pygetwindow as gw except ImportError: return SkillResult(success=False, error="pygetwindow 未安装，请运行: pip install pygetwindow") except NotImplementedError: return SkillResult(success=False, error="当前平台不支持窗口管理（pygetwindow 仅支持 Windows 和 macOS）") if not title: return SkillResult(success=False, error="缺少必需参数: title") try: # 模糊匹配窗口标题 windows = gw.getWindowsWithTitle(title) if not windows: return SkillResult( success=False, error=f"未找到包含 '{title}' 的窗口", ) window = windows[0] if activate: # 先尝试恢复最小化的窗口 if window.isMinimized: window.restore() time.sleep(0.2) window.activate() time.sleep(0.2) if maximize: window.maximize() time.sleep(0.2) return SkillResult( success=True, data={ "title": window.title, "left": window.left, "top": window.top, "width": window.width, "height": window.height, "activated": activate, "maximized": maximize, }, message=f"已聚焦窗口: {window.title} ({window.width}x{window.height})", ) except Exception as e: logger.error(f"窗口聚焦失败: {e}") return SkillResult(success=False, error=f"窗口聚焦失败: {e}") class ScreenElementSkill(Skill): """ 屏幕元素定位 - 使用视觉模型(VLM)从截图中定位 UI 元素坐标。 工作流程: 1. 自动截取屏幕截图 2. 将截图发送到 VLM（视觉语言模型）进行分析 3. VLM 返回目标元素的屏幕坐标 4. 返回坐标供 mouse_click 技能使用 依赖 LLM 客户端配置了支持视觉的模型（如 GPT-4o, Claude 3.5 Sonnet 等）。 """ name = "screen_element" description = ( "使用视觉模型(VLM)从屏幕截图中定位 UI 元素的坐标。" "先截图，再让 AI 视觉模型分析找到目标元素，返回屏幕坐标。" "返回的坐标可直接用于 mouse_click 技能进行点击操作。" ) category = "gui" parameters = [ SkillParameter("description", "string", "要查找的元素的文字描述（如 '保存按钮', '右上角的关闭按钮', '地址栏输入框'）", required=True), SkillParameter("region", "string", "截图区域 'left,top,width,height'（留空截取全屏）", required=False, default=""), SkillParameter("element_type", "string", "元素类型提示: button/input/link/icon/menu/checkbox/text_area/其他", required=False, default=""), ] async def execute( self, description: str = "", region: str = "", element_type: str = "", **kwargs, ) -> SkillResult: """执行：使用 VLM 定位屏幕元素""" # 第一步：截图 try: import mss except Exception: return SkillResult( success=False, error="mss 不可用（当前平台缺少 GUI 环境或未安装）", ) try: screenshot_path = _generate_screenshot_path() # 截图 with mss.mss() as sct: if region: parts = region.split(",") if len(parts) == 4: left, top, width, height = [int(p.strip()) for p in parts] monitor_info = {"left": left, "top": top, "width": width, "height": height} else: monitor_info = sct.monitors[1] else: monitor_info = sct.monitors[1] sct_img = sct.grab(monitor_info) from mss.tools import to_png to_png(sct_img.rgb, sct_img.size, output=screenshot_path) import os file_size = os.path.getsize(screenshot_path) # 第二步：读取图片并编码为 base64 import base64 with open(screenshot_path, "rb") as f: image_data = base64.b64encode(f.read()).decode("utf-8") # 第三步：调用 VLM 分析截图 try: from core.llm import LLMClient, Message client = LLMClient() # 构建 VLM 分析 prompt type_hint = f"元素类型: {element_type}。" if element_type else "" vlm_prompt = ( f"分析这张屏幕截图，找到以下 UI 元素并返回其中心坐标:\n" f"目标: {description}\n" f"{type_hint}\n\n" f"请严格按以下 JSON 格式回复（不要包含其他内容）:\n" f'{{"found": true/false, "x": 中心X坐标, "y": 中心Y坐标, ' f'"confidence": 0.0-1.0, "description": "找到的元素的描述"}}\n\n' f"注意: 坐标以截图左上角为原点 (0,0)。" f"如果找不到元素，found 设为 false。" ) # 使用 vision API messages = [ Message( role="user", content=[ {"type": "text", "text": vlm_prompt}, { "type": "image_url", "image_url": { "url": f"data:image/png;base64,{image_data}", }, }, ], ), ] response = await client.chat( messages=messages, temperature=0.1, max_tokens=200, ) if not response.success or not response.content: return SkillResult( success=True, data={ "screenshot_path": screenshot_path, "file_size_bytes": file_size, "found": False, "reason": "VLM 未返回有效结果", }, message=f"截图已保存: {screenshot_path}，但 VLM 分析失败", files=[screenshot_path], ) # 解析 VLM 返回的坐标 import json import re content = response.content.strip() # 尝试从返回中提取 JSON json_match = re.search(r'\{[^}]+\}', content, re.DOTALL) if json_match: result = json.loads(json_match.group()) else: result = json.loads(content) found = result.get("found", False) x = result.get("x") y = result.get("y") confidence = result.get("confidence", 0) if found and x is not None and y is not None: # 如果截图是区域截图，需要加上偏移量 offset_x = monitor_info.get("left", 0) offset_y = monitor_info.get("top", 0) actual_x = int(x) + offset_x actual_y = int(y) + offset_y return SkillResult( success=True, data={ "found": True, "x": actual_x, "y": actual_y, "confidence": confidence, "description": result.get("description", description), "screenshot_path": screenshot_path, "element_type": element_type, }, message=( f"已定位元素 '{description}' 在坐标 ({actual_x}, {actual_y})，" f"置信度: {confidence:.0%}。可使用 mouse_click 点击。" ), files=[screenshot_path], ) else: return SkillResult( success=True, data={ "found": False, "screenshot_path": screenshot_path, "vlm_response": content[:500], }, message=f"未在截图中找到 '{description}'。截图已保存: {screenshot_path}", files=[screenshot_path], ) except ImportError: return SkillResult( success=True, data={ "screenshot_path": screenshot_path, "file_size_bytes": file_size, "found": False, "reason": "LLM 客户端不可用，无法进行 VLM 分析", }, message=( f"截图已保存: {screenshot_path}，但无法调用 VLM 进行元素定位。" "请检查 LLM 配置是否支持视觉模型，或手动查看截图确定坐标。" ), files=[screenshot_path], ) except Exception as vlm_error: logger.error(f"VLM 分析失败: {vlm_error}") return SkillResult( success=True, data={ "screenshot_path": screenshot_path, "file_size_bytes": file_size, "found": False, "reason": f"VLM 分析异常: {vlm_error}", }, message=( f"截图已保存: {screenshot_path}，但 VLM 分析失败: {vlm_error}。" "可手动查看截图确定坐标后使用 mouse_click。" ), files=[screenshot_path], ) except Exception as e: logger.error(f"屏幕元素定位失败: {e}") return SkillResult(success=False, error=f"屏幕元素定位失败: {e}")