/** * @getpipher/vision — vision extension entry point. * * Registers the `describe_image` tool and the `/vision` slash command, and * keeps the tool's visibility in sync with the active primary model's * capability via `lib/capability.ts` (mechanism A: `setActiveTools`). * * - Multimodal primary → `describe_image` hidden (PASS-THROUGH; native image * reasoning, 0 delegation). A minimal `input` hook in `paste.ts` (B-lite) * guarantees path-referenced images reach the model. * - Text-only primary → `describe_image` visible (DELEGATE to the configured * vision model via `lib/delegate.ts`). * * `/vision` (no arg) opens an interactive settings panel built on pi-tui's * `SettingsList` — the same engine pi's native `/settings` uses. Arrow keys * navigate, Enter cycles a value or opens a sub-picker (e.g. the vision-model * picker), Escape exits. Changes apply live (saved to vision.json + tool * visibility re-synced). Non-TUI modes fall back to a text status. Typed * subcommands (`/vision on`, `/vision model `, …) remain for power users. */ import type { Api, Model } from "@earendil-works/pi-ai"; import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext, Theme } from "@earendil-works/pi-coding-agent"; import { DynamicBorder, getAgentDir } from "@earendil-works/pi-coding-agent"; import { join } from "node:path"; import { Type } from "typebox"; import { StringEnum } from "@earendil-works/pi-ai"; import { Container, type Component, Input, Key, SettingsList, type SettingItem, SelectItem, SelectList, Spacer, Text, } from "@earendil-works/pi-tui"; import { isMultimodal, syncToolAvailability, TOOL_NAME } from "../lib/capability.ts"; import { applySettingChange, DEFAULT_CONFIG, loadConfig, MARKER_STYLES, MAX_BATCH_IMAGES, PASTE_MODES, REASONING_LEVELS, saveConfig, type ReasoningLevel, type VisionConfig, } from "../lib/config.ts"; import { delegateToVisionModel, type DelegateParams } from "../lib/delegate.ts"; import { VisionCache } from "../lib/cache.ts"; import { setSharedState } from "../lib/state.ts"; import { createPreviewComponent, makePreviewImage, detectProtocol, formatImageMetadata } from "../lib/preview.ts"; import { matchesKey } from "@earendil-works/pi-tui"; import { loadImage } from "../lib/image.ts"; import { mapWithConcurrency } from "../lib/batch.ts"; import { buildBatchToolResult } from "../lib/marker.ts"; import { autoDetectDefaults } from "../lib/defaults.ts"; import { clearAuditLog, countAuditLog, resolveAuditPath, tailAuditLog } from "../lib/audit.ts"; /** Current config. Loaded on session_start, mutated by /vision, saved to disk. */ let config: VisionConfig = { ...DEFAULT_CONFIG }; /** Content-addressed delegation cache. Rebuilt on session_start + when * cachePersist/cacheMaxEntries change. Memory-only when cachePersist is off. */ let cache: VisionCache = new VisionCache(undefined, DEFAULT_CONFIG.cacheMaxEntries); function rebuildCache(): void { const dir = config.cachePersist ? join(getAgentDir(), "vision-cache") : undefined; cache = new VisionCache(dir, config.cacheMaxEntries); setSharedState(config, cache); } const SUBCOMMANDS = [ "show", "on", "off", "provider", "model", "max-dim", "quality", "reasoning-effort", "system-prompt", "cache", "fallback", "clear", "paste-mode", "marker-style", "auto-prompt", "preview", "batch-concurrency", "local-only", "audit", "audit-path", ] as const; function formatConfigStatus(c: VisionConfig): string { return [ "Vision tool config:", ` enabled: ${c.enabled}`, ` provider: ${c.provider ?? "(not set)"}`, ` model: ${c.model ?? "(not set)"}`, ` maxDimension: ${c.maxDimension}px`, ` jpegQuality: ${c.jpegQuality}`, ` reasoning: ${c.defaultReasoningEffort}`, ` systemPrompt: ${c.systemPrompt ? truncatePreview(c.systemPrompt, 40) : "(none)"}`, ` cache: ${c.cacheEnabled ? "on" : "off"}${c.cachePersist ? " (persisted, max " + c.cacheMaxEntries + ")" : ""}`, ` retry: ${c.retryAttempts} attempts, ${c.retryBackoffMs}ms backoff`, ` fallback: ${c.fallbackProvider && c.fallbackModel ? c.fallbackProvider + "/" + c.fallbackModel : "(none)"}`, ` markerStyle: ${c.markerStyle}`, ` textOnlyPaste: ${c.textOnlyPasteMode}`, ` autoPrompt: ${c.autoDelegatePrompt ? truncatePreview(c.autoDelegatePrompt, 40) : "(default)"}`, ` autoTimeout: ${c.autoDelegateTimeoutMs}ms`, ` composePreview: ${c.composePreview}`, ` previewMaxWidth: ${c.previewMaxWidthCells} cells`, ` batchConcurrency: ${c.batchConcurrency}`, ` localOnly: ${c.localOnly ? "on" : "off"}`, ` auditLog: ${c.auditLog ? "on" : "off"}`, ` autoDetect: ${c.autoDetectVisionModel ? "on" : "off"}`, ].join("\n"); } /** Truncate a string for a settings-row preview, appending an ellipsis if it overflows. */ function truncatePreview(s: string, max: number): string { const t = s.replace(/\s+/g, " ").trim(); return t.length <= max ? t : `${t.slice(0, max - 1)}…`; } /** Display string for a setting row, from the current config. */ function renderValue(id: string): string { switch (id) { case "enabled": return config.enabled ? "on" : "off"; case "model": return config.provider && config.model ? `${config.provider}/${config.model}` : "(not set)"; case "maxDimension": return `${config.maxDimension}px`; case "jpegQuality": return `${config.jpegQuality}`; case "reasoning": return config.defaultReasoningEffort; case "systemPrompt": return config.systemPrompt ? truncatePreview(config.systemPrompt, 40) : "(none)"; case "cacheEnabled": return config.cacheEnabled ? "on" : "off"; case "cachePersist": return config.cachePersist ? "on" : "off"; case "cacheMaxEntries": return `${config.cacheMaxEntries}`; case "retryAttempts": return `${config.retryAttempts}`; case "retryBackoffMs": return `${config.retryBackoffMs}ms`; case "fallbackModel": return config.fallbackProvider && config.fallbackModel ? `${config.fallbackProvider}/${config.fallbackModel}` : "(none)"; case "markerStyle": return config.markerStyle; case "textOnlyPasteMode": return config.textOnlyPasteMode; case "autoDelegatePrompt": return config.autoDelegatePrompt ? truncatePreview(config.autoDelegatePrompt, 40) : "(default)"; case "autoDelegateTimeoutMs": return `${config.autoDelegateTimeoutMs}ms`; case "composePreview": return config.composePreview ? "on" : "off"; case "previewMaxWidthCells": return `${config.previewMaxWidthCells}`; case "batchConcurrency": return `${config.batchConcurrency}`; case "localOnly": return config.localOnly ? "on" : "off"; case "auditLog": return config.auditLog ? "on" : "off"; case "autoDetectVisionModel": return config.autoDetectVisionModel ? "on" : "off"; default: return ""; } } /** Re-sync tool visibility after a config change that could affect it. */ function resync(pi: ExtensionAPI, ctx: ExtensionCommandContext): void { syncToolAvailability(pi, ctx.model, { enabled: config.enabled }); } /** Apply a setting edit, persist, re-sync visibility if needed, and rebuild * the cache when cache-shape fields change. */ function applyAndSave(id: string, value: string, pi: ExtensionAPI, ctx: ExtensionCommandContext): void { config = applySettingChange(config, id, value); saveConfig(config, getAgentDir()); setSharedState(config, cache); if (id === "enabled" || id === "model") resync(pi, ctx); if (id === "cachePersist" || id === "cacheMaxEntries") rebuildCache(); } /** Vision-capable authed models from the registry (input includes "image"). */ function visionCapableModels(ctx: ExtensionContext): Model[] { return ctx.modelRegistry.getAvailable().filter((m) => m.input.includes("image")); } /** Open pi's native select picker over vision-capable models. Sets provider + * model together. Used by `/vision model` (no arg), `/vision-use`, and the * `alt+shift+v` hotkey as a quick pick. */ async function pickVisionModel(ctx: ExtensionContext): Promise { const models = visionCapableModels(ctx); if (models.length === 0) { ctx.ui.notify( 'No vision-capable models found. Define a model with `input: ["text","image"]` in ~/.pi/agent/models.json and configure its auth.', "warning", ); return false; } const items = models.map((m) => ({ value: `${m.provider}/${m.id}`, label: `${m.provider}/${m.id}` })); let choice: string | undefined; if (ctx.mode === "tui") { // v0.5.1: search-enabled picker in TUI mode (matches pi's /model UX). choice = await ctx.ui.custom((tui, theme, _kb, done) => { const picker = new VisionModelPicker( theme, items, (value) => done(value), () => done(undefined), ); return { render(width: number) { return picker.render(width); }, invalidate() { picker.invalidate(); tui.requestRender(); }, handleInput(data: string) { picker.handleInput(data); tui.requestRender(); }, } as Component & { dispose?(): void }; }); } else { // Non-TUI (RPC/print): fall back to the simple select dialog (no search). choice = await ctx.ui.select("Pick a vision model:", items.map((i) => i.label)); } if (!choice) return false; const slash = choice.indexOf("/"); if (slash <= 0 || slash >= choice.length - 1) return false; config = { ...config, provider: choice.slice(0, slash), model: choice.slice(slash + 1) }; saveConfig(config, getAgentDir()); return true; } /** * Open the interactive `/vision` settings panel (pi-tui SettingsList — the * same engine `/settings` uses). TUI-only; non-TUI modes fall back to text. */ async function showVisionSettings(pi: ExtensionAPI, ctx: ExtensionCommandContext): Promise { if (ctx.mode !== "tui") { ctx.ui.notify(formatConfigStatus(config), "info"); return; } await ctx.ui.custom((tui, theme, _kb, done) => { const container = new Container(); // Blue border lines above + below the panel (matches pi's /settings visual framing). const accentBorder = (text: string) => theme.fg("accent", text); container.addChild(new DynamicBorder(accentBorder)); container.addChild(new Spacer(1)); container.addChild(new Text(theme.fg("accent", theme.bold("Vision tool settings")), 0, 0)); const items: SettingItem[] = [ { id: "enabled", label: "Enabled", currentValue: renderValue("enabled"), values: ["on", "off"], description: "Master switch. Off → describe_image hidden + actionable error if invoked.", }, { id: "model", label: "Vision model", currentValue: renderValue("model"), description: "Model to delegate to (DELEGATE mode). Must have input: [text, image]. Enter opens a picker.", submenu: (_cur, subDone) => buildModelSubmenu(theme, ctx, subDone), }, { id: "maxDimension", label: "Max dimension", currentValue: renderValue("maxDimension"), values: ["512px", "1024px", "1568px", "2048px", "4096px"], description: "Max long-edge pixels for compression.", }, { id: "jpegQuality", label: "JPEG quality", currentValue: renderValue("jpegQuality"), values: ["70", "80", "85", "90", "95"], description: "Re-encode quality (1-100).", }, { id: "reasoning", label: "Reasoning effort", currentValue: renderValue("reasoning"), values: [...REASONING_LEVELS], description: "Default reasoning effort for delegation calls.", }, // ── v0.2.0 (SPEC-2) rows ──────────────────────────────────────────── { id: "systemPrompt", label: "System prompt", currentValue: renderValue("systemPrompt"), description: "Vision-model framing prepended to the request. Enter to edit inline (single-line). For multi-line, use /vision system-prompt.", submenu: (cur, subDone) => buildSystemPromptInput(cur, subDone), }, { id: "cacheEnabled", label: "Caching", currentValue: renderValue("cacheEnabled"), values: ["on", "off"], description: "When on, identical delegation calls return a cached description (0 tokens on hit).", }, { id: "cachePersist", label: "Persist cache to disk", currentValue: renderValue("cachePersist"), values: ["on", "off"], description: "When on, the cache survives session restarts (LRU-evicted at max entries).", }, { id: "cacheMaxEntries", label: "Cache max entries", currentValue: renderValue("cacheMaxEntries"), values: ["64", "128", "256", "512", "1024"], description: "Max disk-cache entries before LRU eviction.", }, { id: "retryAttempts", label: "Retry attempts", currentValue: renderValue("retryAttempts"), values: ["0", "1", "2", "3", "5"], description: "Retries after the first failure (total attempts = this + 1). Only 5xx/429/network retry.", }, { id: "retryBackoffMs", label: "Retry backoff (ms)", currentValue: renderValue("retryBackoffMs"), values: ["250", "500", "1000", "2000"], description: "Base backoff; delay = min(backoffMs * 2^attempt, 8000ms).", }, { id: "fallbackModel", label: "Fallback vision model", currentValue: renderValue("fallbackModel"), description: "Secondary vision model tried when the primary exhausts retries or fails non-retryable. Enter opens a picker.", submenu: (_cur, subDone) => buildModelSubmenu(theme, ctx, subDone), }, // ── v0.3.0 (SPEC-3) rows ──────────────────────────────────────────── { id: "markerStyle", label: "Marker style", currentValue: renderValue("markerStyle"), values: [...MARKER_STYLES], description: "Markdown style for [Image-#N] markers: code (inline code), bold, or plain.", }, { id: "textOnlyPasteMode", label: "Text-only paste mode", currentValue: renderValue("textOnlyPasteMode"), values: [...PASTE_MODES], description: "How pasted images are handled on a text-only primary: hint (nudge to call describe_image), auto (auto-delegate), off (markers only).", }, { id: "autoDelegatePrompt", label: "Auto-delegate prompt", currentValue: renderValue("autoDelegatePrompt"), description: "Generic prompt for auto-delegation in text-only + auto mode. Enter to edit inline (single-line). For multi-line, use /vision auto-prompt.", submenu: (cur, subDone) => buildAutoPromptInput(cur, subDone), }, { id: "autoDelegateTimeoutMs", label: "Auto-delegate timeout", currentValue: renderValue("autoDelegateTimeoutMs"), values: ["10000ms", "20000ms", "30000ms", "60000ms"], description: "Timeout for auto-delegation in the paste hook (per-image AbortController). Falls back to hint on timeout.", }, // ── v0.3.3 (SPEC-3 gap #7) rows ────────────────────────────────────── { id: "composePreview", label: "Compose preview", currentValue: renderValue("composePreview"), values: ["on", "off"], description: "When on, images preview above the editor as you type a path (WhatsApp style). Text fallback on tmux/unsupported terminals.", }, { id: "previewMaxWidthCells", label: "Preview max width", currentValue: renderValue("previewMaxWidthCells"), values: ["40", "60", "80", "100", "120"], description: "Max width (in terminal cells) for the image preview rendering.", }, // ── v0.4.0 (SPEC-4) rows ────────────────────────────────────────── { id: "batchConcurrency", label: "Batch concurrency", currentValue: renderValue("batchConcurrency"), values: ["1", "3", "5", "10", "20"], description: "Max parallel image delegations in a batch (describe_image image_paths + paste auto mode). 1 = serial; 20 = aggressive (rate-limit risk).", }, // ── v0.5.0 (SPEC-5) rows ────────────────────────────────────────── { id: "localOnly", label: "Local-only mode", currentValue: renderValue("localOnly"), values: ["on", "off"], description: "When on, image bytes never leave the machine. Cache hits still work (local); a cache miss refuses with a clear error instead of a network call. Structural guarantee (no network).", }, { id: "auditLog", label: "Audit log", currentValue: renderValue("auditLog"), values: ["on", "off"], description: `When on, every delegation is appended to ${resolveAuditPath(config.auditLogPath, getAgentDir())} (JSONL: provider/model/cached/fallback/ok/error_code). Never logs image bytes or the prompt.`, }, { id: "autoDetectVisionModel", label: "Auto-detect vision model", currentValue: renderValue("autoDetectVisionModel"), values: ["on", "off"], description: "When on + provider/model unset, auto-detect the vision model at session_start (prefers Ollama Cloud primary + a frontier fallback). Persists once; /vision clear re-triggers.", }, ]; const settingsList = new SettingsList( items, 12, { label: (text, selected) => (selected ? theme.fg("accent", theme.bold(text)) : text), value: (text, selected) => (selected ? theme.fg("accent", text) : theme.fg("muted", text)), description: (text) => theme.fg("dim", text), cursor: "❯", hint: (text) => theme.fg("dim", text), }, (id, newValue) => { applyAndSave(id, newValue, pi, ctx); settingsList.updateValue(id, renderValue(id)); }, () => done(true), ); container.addChild(settingsList); container.addChild(new Text(theme.fg("dim", "↑↓ navigate • enter edit/cycle • esc done"), 0, 0)); container.addChild(new Spacer(1)); container.addChild(new DynamicBorder(accentBorder)); return { render(width: number) { return container.render(width); }, invalidate() { container.invalidate(); }, handleInput(data: string) { settingsList.handleInput(data); tui.requestRender(); }, } as Component & { dispose?(): void }; }); } /** * A search-enabled vision-model picker (v0.5.1: RECTOR feedback — match pi's * /model + /settings UX). A Container with blue DynamicBorder lines above + * below, a title, a search Input, + a SelectList that filters as you type. * * Input routing: Escape → cancel; Up/Down/Enter → the SelectList (navigation * + selection); everything else → the search Input, which drives * `selectList.setFilter(input.getValue())` for fuzzy matching. Used by both * the `/vision` panel's "Vision model" submenu (buildModelSubmenu) + the * `/vision model` no-arg quick-pick (pickVisionModel, TUI mode). */ class VisionModelPicker extends Container { private readonly input: Input; private readonly selectList: SelectList; private readonly accentBorder: (text: string) => string; constructor( theme: Theme, items: SelectItem[], onSelect: (value: string) => void, onCancel: () => void, ) { super(); this.accentBorder = (text: string) => theme.fg("accent", text); this.addChild(new DynamicBorder(this.accentBorder)); this.addChild(new Spacer(1)); this.addChild(new Text(theme.bold(theme.fg("accent", "Pick a vision model")), 0, 0)); this.addChild(new Spacer(1)); this.addChild(new Text(theme.fg("muted", "Type to search • ↑↓ navigate • enter select • esc cancel"), 0, 0)); this.addChild(new Spacer(1)); this.input = new Input(); this.input.onSubmit = () => { const item = this.selectList.getSelectedItem(); if (item) onSelect(item.value); }; this.input.onEscape = () => onCancel(); this.addChild(this.input); this.addChild(new Spacer(1)); this.selectList = new SelectList(items, 10, { selectedPrefix: (text) => theme.fg("accent", text), selectedText: (text) => theme.fg("accent", text), description: (text) => theme.fg("muted", text), scrollInfo: (text) => theme.fg("dim", text), noMatch: (text) => theme.fg("warning", text), }); this.selectList.onSelect = (item) => onSelect(item.value || ""); this.selectList.onCancel = () => onCancel(); this.addChild(this.selectList); this.addChild(new Spacer(1)); this.addChild(new DynamicBorder(this.accentBorder)); } handleInput(data: string): void { // Navigation + selection → the SelectList. if (matchesKey(data, "up") || matchesKey(data, "down") || matchesKey(data, "enter") || matchesKey(data, "return")) { this.selectList.handleInput(data); this.invalidate(); return; } // Escape → cancel (let the Input's onEscape handle it, but also guard here). if (matchesKey(data, "escape") || matchesKey(data, "esc")) { this.selectList.onCancel?.(); this.invalidate(); return; } // Everything else → the search Input, then re-filter the list. this.input.handleInput(data); this.selectList.setFilter(this.input.getValue()); this.invalidate(); } } /** Build the vision-model sub-picker shown when Enter is pressed on the model * row of the settings panel. A SelectList over vision-capable authed models. */ function buildModelSubmenu( theme: Theme, ctx: ExtensionCommandContext, subDone: (selectedValue?: string) => void, ): Component { const models = visionCapableModels(ctx); const items: SelectItem[] = models.length > 0 ? models.map((m) => ({ value: `${m.provider}/${m.id}`, label: `${m.provider}/${m.id}` })) : [{ value: "", label: "(no vision-capable models — add one to models.json)" }]; // v0.5.1: search-enabled picker (matches pi's /model + /settings UX). return new VisionModelPicker( theme, items, (value) => subDone(value || undefined), () => subDone(), ); } /** Build the single-line system-prompt editor shown when Enter is pressed on * the system-prompt row. An `Input` (the same component SettingsList uses * for its own search box). Empty submit clears; Escape cancels. */ function buildSystemPromptInput( currentValue: string, subDone: (selectedValue?: string) => void, ): Component { const input = new Input(); input.setValue(currentValue === "(none)" ? "" : currentValue); input.onSubmit = (value) => subDone(value); // "" commits → applySettingChange clears input.onEscape = () => subDone(); // undefined → cancel (no change) return input; } /** Build the single-line auto-delegate-prompt editor (mirrors buildSystemPromptInput). */ function buildAutoPromptInput( currentValue: string, subDone: (selectedValue?: string) => void, ): Component { const input = new Input(); input.setValue(currentValue === "(default)" ? "" : currentValue); input.onSubmit = (value) => subDone(value); input.onEscape = () => subDone(); return input; } /** Normalize the model's image_path / image_paths args into a deduped * `string[]`. Schema-tolerant: some models (Opus 4.6, GLM-5.1) send arrays * as a JSON string (edit.js:36 precedent) — coerce that. Accepts * `string | string[] | undefined` for both fields. Merges image_paths first * (the batch field) then image_path, filters empties, dedups case-sensitively * preserving first-occurrence order. (SPEC-4 §3.1, PLAN-4 §1.1/§1.6.) */ export function normalizeImagePaths(params: { image_path?: string | string[]; image_paths?: string | string[]; }): string[] { const coerce = (v: string | string[] | undefined): string[] => { if (v === undefined) return []; if (Array.isArray(v)) return v.filter((x) => typeof x === "string"); if (typeof v !== "string") return []; const s = v.trim(); if (s === "") return []; // Maybe a JSON-stringified array (some models send arrays as JSON strings) if (s.startsWith("[")) { try { const parsed = JSON.parse(s); if (Array.isArray(parsed)) return parsed.filter((x) => typeof x === "string"); } catch { // not valid JSON → treat as a single path string } } return [s]; }; const merged = [...coerce(params.image_paths), ...coerce(params.image_path)]; const seen = new Set(); const out: string[] = []; for (const p of merged) { const t = typeof p === "string" ? p.trim() : ""; if (t.length === 0) continue; if (seen.has(t)) continue; // dedup, first occurrence wins seen.add(t); out.push(t); } return out; } export default function visionExtension(pi: ExtensionAPI): void { // ── Session lifecycle ─────────────────────────────────────────────────── pi.on("session_start", (_event, ctx) => { config = loadConfig(getAgentDir()); // ── Auto-detect workflow-fit defaults (SPEC-5 §3.3) ─────────────── // Fires only when BOTH provider + model are unset (fresh config). A // partial config (one set, one blank) is the user mid-configuration — // don't overwrite. The detected values are persisted once (the user sees // them + can override; /vision clear re-triggers). Prefers the Ollama // provider's vision models (AGENTS.md "Ollama Cloud primary") + a // frontier fallback (first non-Ollama vision model). if (config.autoDetectVisionModel && !config.provider && !config.model && typeof ctx.modelRegistry?.getAvailable === "function") { const detected = autoDetectDefaults(ctx.modelRegistry.getAvailable()); if (detected.provider && detected.model) { // v0.5.1: auto-detect sets ONLY the primary. The fallback is NOT // auto-populated — the user sets it explicitly via /vision fallback // if they want frontier escalation. (RECTOR feedback: a default // fallback was too opinionated.) config = { ...config, provider: detected.provider, model: detected.model, }; saveConfig(config, getAgentDir()); ctx.ui.notify( `Vision: auto-configured ${detected.provider}/${detected.model}. /vision to change.`, "info", ); } } rebuildCache(); setSharedState(config, cache); syncToolAvailability(pi, ctx.model, { enabled: config.enabled }); }); // Re-sync when the user switches models mid-session (/model, Ctrl+P, restore). pi.on("model_select", (event) => { syncToolAvailability(pi, event.model, { enabled: config.enabled }); }); // v0.1.0 holds no background resources; shutdown is a no-op (forward-compat // with SPEC-2 retry/caching state). pi.on("session_shutdown", () => {}); // ── describe_image tool ───────────────────────────────────────────────── // Always registered; visibility is gated by syncToolAvailability so // multimodal models never see it (0 delegation is structurally impossible). pi.registerTool({ name: TOOL_NAME, label: "Describe Image", description: "Analyze one or more image files and return text descriptions or answer questions about them. Delegates to a configured vision model when the active primary model cannot process images natively. Accepts file paths, data URLs, or raw base64. For multiple images (comparison, cross-reference), pass image_paths (up to 50).", promptSnippet: "Analyze one or more image files and return text descriptions or answer questions about them", promptGuidelines: [ "Use describe_image when you need to analyze an image file and the active model cannot process images natively. describe_image delegates to a configured vision model and returns its text response. For 2+ images, pass image_paths for batch analysis (parallel, one call).", ], parameters: Type.Object({ image_path: Type.Optional( Type.String({ description: "Path to a single image file, a data: URL, or raw base64. Use this for one image. For multiple images, prefer image_paths.", }), ), image_paths: Type.Optional( Type.Array(Type.String(), { description: "Multiple image paths/data URLs/base64 strings to analyze together (e.g. for comparison or cross-reference). Use this for 2+ images. Up to 50 images per call.", }), ), prompt: Type.String({ description: "What to analyze, extract, or answer about the image(s). For multiple images, this prompt applies to each; describe what to compare or how they relate.", }), compress: Type.Optional( Type.Boolean({ description: "Optimize (resize + re-encode) the image(s) before delegation. Default true.", }), ), reasoning: Type.Optional( StringEnum([...REASONING_LEVELS], { description: "Reasoning effort for the delegation call(s). Defaults to the configured defaultReasoningEffort.", }), ), }), async execute(_toolCallId, params, signal, _onUpdate, ctx) { // Defense-in-depth: under mechanism (A) the tool is hidden from // multimodal models, so this branch should never fire. It handles the // rare race where the model switched between the LLM deciding to call // the tool and the tool executing. if (isMultimodal(ctx.model)) { const id = ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : "unknown"; return { content: [ { type: "text" as const, text: `The active primary model (${id}) can process images natively. Use the \`read\` tool to view the image, then respond directly — no delegation needed.`, }, ], details: { mode: "passthrough_redirect", model: id }, }; } // v0.4.0: normalize + validate paths (image_path | image_paths). const paths = normalizeImagePaths(params); if (paths.length === 0) { return { content: [{ type: "text" as const, text: "Vision tool error: describe_image requires image_path or image_paths (got neither)." }], details: { mode: "delegate", error: "no_image_path" }, isError: true, }; } if (paths.length > MAX_BATCH_IMAGES) { return { content: [{ type: "text" as const, text: `Vision tool error: describe_image received ${paths.length} images; the batch cap is ${MAX_BATCH_IMAGES}. Split across multiple calls.` }], details: { mode: "delegate", error: "batch_too_large" }, isError: true, }; } const reasoning = (params.reasoning ?? config.defaultReasoningEffort) as ReasoningLevel; const compress = params.compress ?? true; // Single-image back-compat path (v0.3.x behavior, byte-for-byte). if (paths.length === 1) { const result = await delegateToVisionModel(ctx, config, { image_path: paths[0]!, prompt: params.prompt, compress, reasoning }, signal, cache); if (result.ok) { return { content: [{ type: "text" as const, text: result.text }], details: { mode: "delegate", ...result.details } }; } return { content: [{ type: "text" as const, text: result.error.message }], details: { mode: "delegate", error: result.error.code }, isError: true, }; } // Batch path: parallel, bounded by batchConcurrency, per-image resilience // (fn wraps to sentinel — a failed image becomes an [error: …] section, // never a whole-batch reject). Uses ctx.signal (defined during the agent // run — verified PLAN-3 §1.3). No extra batch timeout. const batchResults = await mapWithConcurrency(paths, config.batchConcurrency, async (p) => { try { const r = await delegateToVisionModel(ctx, config, { image_path: p, prompt: params.prompt, compress, reasoning }, signal, cache); if (r.ok) { return { ok: true as const, text: r.text, cached: r.details.cached, fallback: r.details.fallback, fallbackModel: r.details.fallback ? r.details.model : undefined }; } return { ok: false as const, errorCode: r.error.code, message: r.error.message }; } catch (err) { // belt-and-suspenders: delegateToVisionModel doesn't throw, but guard return { ok: false as const, errorCode: "unexpected", message: err instanceof Error ? err.message : String(err) }; } }); const text = buildBatchToolResult(paths, batchResults); const allFailed = batchResults.every((r) => !r.ok); return { content: [{ type: "text" as const, text }], details: { mode: "delegate-batch", batch: batchResults.map((r, i) => ({ index: i, path: paths[i]!, ok: r.ok, cached: r.ok ? r.cached : false, fallback: r.ok ? r.fallback : false, errorCode: r.ok ? undefined : r.errorCode, })), }, // Only flag isError when EVERY image failed (matches single-path back-compat, // which omits isError on success). A partial failure is NOT a whole-batch error. ...(allFailed ? { isError: true as const } : {}), }; }, }); // ── /vision slash command ────────────────────────────────────────────── pi.registerCommand("vision", { description: "Open the vision settings panel (like /settings). Subcommands: show, on, off, provider

, model [], max-dim , quality <1-100>, reasoning-effort , system-prompt [|clear], cache , fallback [|clear>, clear.", handler: async (args, ctx) => { const parts = args.trim().split(/\s+/).filter(Boolean); const sub = parts[0] ?? ""; // empty → open the settings panel const agentDir = getAgentDir(); switch (sub) { case "": { // /vision (no arg) → interactive settings panel (TUI) or text status. await showVisionSettings(pi, ctx); return; } case "show": { ctx.ui.notify(formatConfigStatus(config), "info"); return; } case "on": { config = { ...config, enabled: true }; saveConfig(config, agentDir); resync(pi, ctx); ctx.ui.notify("Vision tool enabled.", "info"); return; } case "off": { config = { ...config, enabled: false }; saveConfig(config, agentDir); resync(pi, ctx); ctx.ui.notify("Vision tool disabled. Use /vision on to re-enable.", "info"); return; } case "provider": { const value = parts[1]; if (!value) { ctx.ui.notify("Usage: /vision provider ", "warning"); return; } config = { ...config, provider: value }; saveConfig(config, agentDir); resync(pi, ctx); ctx.ui.notify(`Vision provider set to ${value}.`, "info"); return; } case "model": { const value = parts.slice(1).join(" ").trim(); if (!value) { // Quick pick via native select dialog (works in RPC too). const picked = await pickVisionModel(ctx); if (picked) { resync(pi, ctx); ctx.ui.notify(`Vision model set to ${config.provider}/${config.model}.`, "info"); } return; } config = { ...config, model: value }; saveConfig(config, agentDir); resync(pi, ctx); ctx.ui.notify(`Vision model set to ${value}.`, "info"); return; } case "max-dim": { const n = Number(parts[1]); if (!Number.isFinite(n)) { ctx.ui.notify("Usage: /vision max-dim ", "warning"); return; } config = { ...config, maxDimension: Math.min(8000, Math.max(1, Math.round(n))) }; saveConfig(config, agentDir); ctx.ui.notify(`Max dimension set to ${config.maxDimension}px.`, "info"); return; } case "quality": { const n = Number(parts[1]); if (!Number.isFinite(n)) { ctx.ui.notify("Usage: /vision quality <1-100>", "warning"); return; } config = { ...config, jpegQuality: Math.min(100, Math.max(1, Math.round(n))) }; saveConfig(config, agentDir); ctx.ui.notify(`JPEG quality set to ${config.jpegQuality}.`, "info"); return; } case "reasoning-effort": { const raw = parts[1]; if (!raw || !(REASONING_LEVELS as readonly string[]).includes(raw)) { ctx.ui.notify( `Usage: /vision reasoning-effort <${REASONING_LEVELS.join("|")}>`, "warning", ); return; } config = { ...config, defaultReasoningEffort: raw as ReasoningLevel }; saveConfig(config, agentDir); ctx.ui.notify(`Default reasoning effort set to ${raw}.`, "info"); return; } case "clear": { config = { ...DEFAULT_CONFIG }; saveConfig(config, agentDir); rebuildCache(); resync(pi, ctx); ctx.ui.notify("Vision config reset to defaults.", "info"); return; } case "system-prompt": { const value = parts.slice(1).join(" ").trim(); if (!value) { // No arg → multi-line editor (safe: command handler, not inside ctx.ui.custom). if (ctx.hasUI) { const edited = await ctx.ui.editor("Vision system prompt", config.systemPrompt ?? ""); if (edited === undefined) return; // cancelled config = { ...config, systemPrompt: edited.trim().length > 0 ? edited.trim() : undefined }; } else { ctx.ui.notify("Usage: /vision system-prompt (or /vision system-prompt clear)", "warning"); return; } } else if (value === "clear") { config = { ...config, systemPrompt: undefined }; } else { config = { ...config, systemPrompt: value }; } saveConfig(config, agentDir); ctx.ui.notify(config.systemPrompt ? "Vision system prompt set." : "Vision system prompt cleared.", "info"); return; } case "cache": { const action = parts[1]; if (action === "clear") { cache.clear(); ctx.ui.notify("Vision cache cleared (memory + disk).", "info"); } else if (action === "show") { const s = cache.stats(); ctx.ui.notify(`Vision cache: ${s.memoryEntries} memory, ${s.diskEntries} disk (max ${s.maxEntries}, persisted ${s.persisted}). Memory is session-scoped; enable \"Persist cache to disk\" for cross-session hits.`, "info"); } else { ctx.ui.notify("Usage: /vision cache ", "warning"); } return; } case "fallback": { const value = parts.slice(1).join(" ").trim(); if (!value) { ctx.ui.notify("Usage: /vision fallback (or /vision fallback clear)", "warning"); return; } if (value === "clear") { config = { ...config, fallbackProvider: undefined, fallbackModel: undefined }; saveConfig(config, agentDir); ctx.ui.notify("Fallback vision model cleared.", "info"); return; } const slash = value.indexOf("/"); if (slash > 0 && slash < value.length - 1) { config = { ...config, fallbackProvider: value.slice(0, slash), fallbackModel: value.slice(slash + 1) }; } else { config = { ...config, fallbackModel: value }; } saveConfig(config, agentDir); ctx.ui.notify(`Fallback vision model set to ${config.fallbackProvider}/${config.fallbackModel}.`, "info"); return; } case "paste-mode": { const value = parts[1]; if (!value) { const order = PASTE_MODES as readonly string[]; const next = order[(order.indexOf(config.textOnlyPasteMode) + 1) % order.length] ?? "hint"; config = applySettingChange(config, "textOnlyPasteMode", next); } else { config = applySettingChange(config, "textOnlyPasteMode", value); } saveConfig(config, agentDir); setSharedState(config, cache); ctx.ui.notify(`Text-only paste mode set to ${config.textOnlyPasteMode}.`, "info"); return; } case "marker-style": { const value = parts[1]; if (!value) { ctx.ui.notify(`Marker style: ${config.markerStyle}. Valid: ${MARKER_STYLES.join(", ")}`, "info"); return; } config = applySettingChange(config, "markerStyle", value); saveConfig(config, agentDir); setSharedState(config, cache); ctx.ui.notify( config.markerStyle === value ? `Marker style set to ${value}.` : `Invalid style. Valid: ${MARKER_STYLES.join(", ")}`, config.markerStyle === value ? "info" : "warning", ); return; } case "auto-prompt": { const value = parts.slice(1).join(" ").trim(); if (!value) { if (ctx.hasUI) { const edited = await ctx.ui.editor("Auto-delegate prompt", config.autoDelegatePrompt === DEFAULT_CONFIG.autoDelegatePrompt ? "" : config.autoDelegatePrompt); if (edited === undefined) return; config = applySettingChange(config, "autoDelegatePrompt", edited); } else { ctx.ui.notify("Usage: /vision auto-prompt (or /vision auto-prompt clear)", "warning"); return; } } else if (value === "clear") { config = applySettingChange(config, "autoDelegatePrompt", ""); } else { config = applySettingChange(config, "autoDelegatePrompt", value); } saveConfig(config, agentDir); setSharedState(config, cache); ctx.ui.notify(config.autoDelegatePrompt === DEFAULT_CONFIG.autoDelegatePrompt ? "Auto-delegate prompt reset to default." : "Auto-delegate prompt set.", "info"); return; } case "preview": { const path = parts.slice(1).join(" ").trim(); if (!path) { ctx.ui.notify("Usage: /vision preview ", "warning"); return; } // Clear compose preview widget if active (prevent interference) if (typeof ctx.ui.setWidget === "function") { ctx.ui.setWidget("vision-compose-preview", undefined); } // Helpful error if the user passed a marker instead of a path if (path.includes("[Image-#") || path.startsWith("`")) { ctx.ui.notify("Vision preview: pass a file path, not a marker. Example: /vision preview /tmp/screenshot.png", "warning"); return; } if (ctx.mode !== "tui") { // Non-TUI: notify metadata as text const loaded = await loadImage(path, { compress: false, maxDimension: 1568, jpegQuality: 85, cwd: ctx.cwd }); if (!loaded.ok) { ctx.ui.notify(`Vision preview error: could not load image "${path}".`, "error"); return; } const img = makePreviewImage(loaded.image.data, loaded.image.mimeType, path); ctx.ui.notify(formatImageMetadata(img, detectProtocol()), "info"); return; } // TUI: open a custom panel with the Image component const loaded = await loadImage(path, { compress: false, maxDimension: 1568, jpegQuality: 85, cwd: ctx.cwd }); if (!loaded.ok) { ctx.ui.notify(`Vision preview error: could not load image "${path}" (${loaded.error.code}).`, "error"); return; } const img = makePreviewImage(loaded.image.data, loaded.image.mimeType, path); await ctx.ui.custom((_tui, theme, keybindings, done) => { const component = createPreviewComponent(img, (c: string, t: string) => theme.fg(c as any, t), config.previewMaxWidthCells); return { render(width: number) { return component.render(width); }, invalidate() { component.invalidate(); }, handleInput(data: string) { // Close on Escape if (data === "\x1b" || matchesKey(data, "escape") || matchesKey(data, "esc")) { done(undefined); } }, dispose() { component.dispose?.(); }, } as Component & { dispose?(): void }; }); return; } case "batch-concurrency": { const raw = parts[1]; if (!raw) { ctx.ui.notify(`Batch concurrency: ${config.batchConcurrency} (1–20). 1 = serial, 20 = aggressive.`, "info"); return; } const n = parseInt(raw, 10); if (!Number.isFinite(n)) { ctx.ui.notify("Usage: /vision batch-concurrency <1-20>", "warning"); return; } config = applySettingChange(config, "batchConcurrency", String(n)); saveConfig(config, agentDir); setSharedState(config, cache); ctx.ui.notify(`Batch concurrency set to ${config.batchConcurrency}.`, "info"); return; } case "local-only": { const value = parts[1]; if (!value) { ctx.ui.notify( `Local-only mode: ${config.localOnly ? "on" : "off"}. ` + "When on, image bytes never leave the machine (cache hits still work; a cache miss refuses with a clear error instead of a network call). " + "Toggle via /vision local-only on|off.", "info", ); return; } if (value !== "on" && value !== "off") { ctx.ui.notify("Usage: /vision local-only ", "warning"); return; } config = applySettingChange(config, "localOnly", value); saveConfig(config, agentDir); setSharedState(config, cache); ctx.ui.notify(`Local-only mode ${config.localOnly ? "enabled" : "disabled"}.`, "info"); return; } case "audit": { const action = parts[1]; const path = resolveAuditPath(config.auditLogPath, agentDir); if (action === "clear") { clearAuditLog(path); ctx.ui.notify(`Audit log cleared (${path}).`, "info"); return; } if (action === "show") { const entries = tailAuditLog(path, 10); const total = countAuditLog(path); const lines = entries.map((e) => `[${e.ts}] ${e.provider}/${e.model} ${e.cached ? "(cached)" : e.fallback ? "(fallback)" : ""} ok=${e.ok}${e.error_code ? ` err=${e.error_code}` : ""}${e.local_only ? " local-only" : ""} ${e.latency_ms}ms ${e.image_path}`, ); ctx.ui.notify( `Audit log (${path}) - ${total} entries, last 10:\n${lines.join("\n") || "(empty)"}`, "info", ); return; } if (action === "path") { ctx.ui.notify(`Audit log path: ${path}`, "info"); return; } if (action === "on" || action === "off") { config = applySettingChange(config, "auditLog", action); saveConfig(config, agentDir); setSharedState(config, cache); ctx.ui.notify(`Audit logging ${config.auditLog ? "on" : "off"} (${path}).`, "info"); return; } ctx.ui.notify("Usage: /vision audit ", "warning"); return; } case "audit-path": { const value = parts.slice(1).join(" ").trim(); if (!value) { ctx.ui.notify(`Audit log path: ${resolveAuditPath(config.auditLogPath, agentDir)}${config.auditLogPath ? " (custom)" : " (default)"}`, "info"); return; } if (value === "clear") { config = applySettingChange(config, "auditLogPath", "clear"); } else { config = applySettingChange(config, "auditLogPath", value); } saveConfig(config, agentDir); setSharedState(config, cache); ctx.ui.notify( `Audit log path set to ${resolveAuditPath(config.auditLogPath, agentDir)}.`, "info", ); return; } default: { ctx.ui.notify( `Unknown /vision subcommand: ${sub}\nAvailable: ${SUBCOMMANDS.join(", ")} (or just /vision for the panel)`, "warning", ); } } }, }); // ── /vision-use command + ctrl+shift+i hotkey (SPEC-2 gap #5: inline switch) ─ // Both switch the DELEGATE vision model mid-session without the full panel. // Tool visibility is unaffected (it tracks the PRIMARY model's capability, // not the vision model) so no resync is needed. The hotkey uses ctrl+shift+i // (not alt+) so it works on Mac terminals where Option≠Alt by default // (e.g. Ghostty macos-option-as-alt=false). Rebindable via keybindings.json. pi.registerCommand("vision-use", { description: "Switch the DELEGATE vision model inline. No arg → picker; → set directly. (Hotkey: ctrl+shift+i)", handler: async (args, ctx) => { const value = args.trim(); if (!value) { const picked = await pickVisionModel(ctx); if (picked) ctx.ui.notify(`Vision model set to ${config.provider}/${config.model}.`, "info"); return; } const slash = value.indexOf("/"); if (slash > 0 && slash < value.length - 1) { config = { ...config, provider: value.slice(0, slash), model: value.slice(slash + 1) }; } else { config = { ...config, model: value }; } saveConfig(config, getAgentDir()); ctx.ui.notify(`Vision model set to ${config.provider}/${config.model}.`, "info"); }, }); pi.registerShortcut(Key.ctrlShift("i"), { description: "Switch vision model (inline picker)", handler: async (ctx) => { const picked = await pickVisionModel(ctx); if (picked) ctx.ui.notify(`Vision model set to ${config.provider}/${config.model}.`, "info"); }, }); }