import { completeSimple as defaultCompleteSimple } from "@earendil-works/pi-ai/compat"; import type { ExtensionAPI, ToolDefinition, } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; import { getAgentRuntimeComposition } from "../../shared/agent-runtime-composition"; import { readSuiteConfigFile } from "../../shared/agent-suite-storage"; import type { AuxiliaryLlmCompletion } from "../../shared/auxiliary-llm"; import { registerPackageTool } from "../../shared/tool-presentation/registry"; import { publishUsageEvent } from "../../shared/usage-events"; import { parseVisionConfig, type VisionConfig } from "./config"; import { describeImage, resolveVisionRuntime } from "./delegate"; import { ImageLoadError, loadImage } from "./image"; import { renderVisionCall, renderVisionResult } from "./rendering"; const TOOL_NAME = "describe_image"; const EXTENSION_DIRECTORY = "vision"; const ERROR_PREFIX_PATTERN = /^[^:]+:\s*/; const TOOL_PARAMETERS = Type.Object( { image_path: Type.String({ minLength: 1, description: "Path to image file.", }), prompt: Type.String({ minLength: 1, maxLength: 2048, description: "Question or instruction to answer about image. MUST use ASD-STE100 - Simplified Technical English, unless maintaining original language is essential for accuracy.", }), }, { additionalProperties: false }, ); interface VisionParams { readonly image_path: string; readonly prompt: string; } type VisionExecutionContext = Parameters< NonNullable["execute"]> >[4]; export function isMultimodal( model: { readonly input?: readonly string[] } | undefined, ): boolean { return model?.input?.includes("image") === true; } export default function vision( pi: ExtensionAPI, dependencies: { readonly completeSimple?: AuxiliaryLlmCompletion; readonly readConfigFile?: typeof readSuiteConfigFile; } = {}, ): void { const completeSimple = dependencies.completeSimple ?? defaultCompleteSimple; const readConfigFile = dependencies.readConfigFile ?? readSuiteConfigFile; let config = defaultConfig(); const sync = createToolSynchronizer(pi, () => config); pi.on("session_start", async (_event, ctx) => { config = await loadConfig(ctx.ui, readConfigFile); sync(ctx.model); }); pi.on("model_select", (event) => sync(event.model)); registerPackageTool( pi, createToolDefinition(pi, () => config, completeSimple), ); getAgentRuntimeComposition(pi).publishBaselineToolNames([TOOL_NAME]); } function createToolSynchronizer( pi: ExtensionAPI, getConfig: () => VisionConfig, ) { return (model: { readonly input?: readonly string[] } | undefined): void => { const config = getConfig(); const available = config.enabled && config.model?.id !== undefined && !isMultimodal(model); getAgentRuntimeComposition(pi).setRestrictiveToolFilter( "vision-availability", (candidates) => candidates.filter((name) => name !== TOOL_NAME || available), ); }; } function createToolDefinition( pi: ExtensionAPI, getConfig: () => VisionConfig, completeSimple: AuxiliaryLlmCompletion, ): ToolDefinition { return { name: TOOL_NAME, label: "Describe Image", description: "Analyze one image file with a vision model. You cannot see images directly - read tool will not show them to you, so use describe_image instead of read for any image question. Each call is independent and does not remember earlier calls; ask everything about image in one prompt.", promptSnippet: "Analyze one image file and return a text description or answer questions about it", promptGuidelines: [ "Use describe_image instead of read for any image; your model cannot view images directly.", ], parameters: TOOL_PARAMETERS, executionMode: "sequential" as const, renderCall: renderVisionCall, renderResult: renderVisionResult, async execute(...[_toolCallId, params, signal, _onUpdate, ctx]) { return executeVisionCall({ params: params as VisionParams, signal, ctx, config: getConfig(), completeSimple, pi, }); }, }; } async function executeVisionCall(options: { readonly params: VisionParams; readonly signal: AbortSignal | undefined; readonly ctx: VisionExecutionContext; readonly config: VisionConfig; readonly completeSimple: AuxiliaryLlmCompletion; readonly pi: ExtensionAPI; }) { if (isMultimodal(options.ctx.model)) { return toolResult("Use read tool for image analysis."); } if (options.config.model?.id === undefined) { throw new Error("not_configured: model.id must be configured"); } const resolved = await resolveVisionRuntime( options.ctx, options.config.model.id, options.config.model.thinking, ); try { const image = await loadImage(options.params.image_path, { cwd: options.ctx.cwd, compression: options.config.compression, }); return toolResult( await describeImage({ runtime: resolved.runtime, thinking: resolved.thinking, image, prompt: options.params.prompt, retry: options.config.retry, signal: options.signal, completeSimple: options.completeSimple, onComplete: (message) => publishUsageEvent(options.pi, "vision", message), }), ); } catch (error) { const { code, message } = formatImageError(error); return toolResult(`[error: ${code} — ${message}]`); } } async function loadConfig( ui: { notify(message: string, level: "warning"): void } | undefined, readConfigFile: typeof readSuiteConfigFile, ): Promise { const file = await readConfigFile(EXTENSION_DIRECTORY); const parsed = file.kind === "found" ? parseJsonConfig(file.file.content) : parseVisionConfig({}); if (parsed.kind === "invalid") { ui?.notify(`[vision] ${parsed.issue}`, "warning"); return defaultConfig(); } if (parsed.config.enabled && parsed.config.model?.id === undefined) { ui?.notify("[vision] model.id must be configured", "warning"); } return parsed.config; } function toolResult(text: string) { return { content: [{ type: "text" as const, text }], details: {} }; } function formatImageError(error: unknown): { readonly code: string; readonly message: string; } { if (error instanceof ImageLoadError) { return { code: error.code, message: error.message }; } if (error instanceof Error) { const [code] = error.message.split(":"); return { code: code ?? "error", message: error.message.replace(ERROR_PREFIX_PATTERN, ""), }; } return { code: "error", message: String(error) }; } function parseJsonConfig(content: string) { try { return parseVisionConfig(JSON.parse(content)); } catch { return { kind: "invalid" as const, issue: "config contains invalid JSON" }; } } function defaultConfig(): VisionConfig { const result = parseVisionConfig({}); if (result.kind === "invalid") { throw new Error(result.issue); } return result.config; }