"use strict"; import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; import { getConfig } from "../config"; import { processImage } from "../media"; import { visionCompletion } from "../client"; import { TEXT_EXTRACTION } from "../prompts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, truncateHead, } from "@earendil-works/pi-coding-agent"; export function registerExtractTextTool(pi: ExtensionAPI) { pi.registerTool({ name: "extract_text_from_screenshot", label: "Extract Text", description: "Extract and transcribe all visible text from a screenshot with maximum accuracy. Preserves original formatting, indentation, and structure. Ideal for code extraction, terminal output, config files, and documentation screenshots. Supports local files (png, jpg) and HTTP(S) URLs.", promptSnippet: "Extract text/code from screenshots with OCR — preserves indentation and structure", promptGuidelines: [ "Use extract_text_from_screenshot when the user provides a screenshot containing code, terminal output, logs, config files, or documentation that needs to be transcribed as text. This tool also accepts an optional programmingLanguage hint for better code extraction accuracy.", ], parameters: Type.Object({ imagePath: Type.String({ description: "Path to the screenshot (relative to cwd, supports @ prefix) or HTTP(S) URL.", }), prompt: Type.Optional( Type.String({ description: "Specific extraction instructions. If omitted, defaults to 'Extract all visible text from this screenshot.'", }) ), programmingLanguage: Type.Optional( Type.String({ description: "Programming language hint for better code extraction accuracy (e.g., 'python', 'rust', 'typescript').", }) ), }), async execute(_toolCallId, params, signal, onUpdate, ctx) { const config = getConfig(); onUpdate?.({ content: [{ type: "text", text: "正在调用 GLM-4.6V 提取文字..." }], }); const image = processImage(params.imagePath, ctx.cwd, config.maxImageSizeMB); // 构建 user prompt let userPrompt = params.prompt || "Extract all visible text from this screenshot."; if (params.programmingLanguage) { userPrompt = `Programming language context: ${params.programmingLanguage}\n\n${userPrompt}`; } const result = await visionCompletion( config, TEXT_EXTRACTION, [image], userPrompt, signal ); const truncated = truncateHead(result, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES, }); let text = truncated.content; if (truncated.truncated) { text += `\n\n[输出已截断: ${truncated.outputLines}/${truncated.totalLines} 行]`; } return { content: [{ type: "text", text }], details: { raw: result }, }; }, }); }