import type { Api, AssistantMessage, ImageContent, Model } from "@earendil-works/pi-ai"; // ponytail: "@earendil-works/pi-ai/compat" is documented as temporary ("deleted with the // coding-agent ModelManager migration"), but it is what pi's own examples/extensions/summarize.ts // uses and the only entry exposing complete(). Swap for the successor API when it lands. import { complete } from "@earendil-works/pi-ai/compat"; import type { AgentToolResult, ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; import { decodedBase64Length, MAX_ATTACHMENT_COUNT, MAX_IMAGE_BYTES, readImageFile } from "./image-file.js"; import { resolveVisionModel, type VisionAuth } from "./vision-model.js"; export const TOOL_NAME = "image_ask"; export interface ImageAskDetails { model: string; imageCount: number; } export interface VisionAnalysis extends ImageAskDetails { text: string; usage: AssistantMessage["usage"]; } /** Injected so tests never touch the network. */ export type CompleteFn = typeof complete; const parameters = Type.Object({ paths: Type.Array(Type.String(), { description: "Local image paths, absolute or relative to cwd; HTTP(S) URLs are rejected — download the image first. Each file may be at most 20 MiB; the format is identified by the file header, not the extension. At most 4 images per call, 20 MiB total. Group related images for comparison.", }), question: Type.String({ description: "A specific question about the images." }), }); /** * Minimum length for a header/env value to be treated as a credential when * scanning model output. Short values ("prod", "Bearer", a team name) are * ordinary words that appear in normal answers; redacting whole responses over * them blinded the tool randomly depending on provider config (BYTE-5 #6). */ const CREDENTIAL_MIN_LENGTH = 8; function credentialEcho(raw: string, auth: VisionAuth): "the API key" | "a configured credential" | undefined { if (auth.apiKey && raw.includes(auth.apiKey)) return "the API key"; // A null header value suppresses a provider default and never carries a credential. const headerEntries = Object.entries(auth.headers ?? {}).filter( (entry): entry is [string, string] => entry[1] !== null, ); // The token part of an Authorization header ("Bearer x" style values are // stripped of the scheme first), subject to the same length bar — an // Authorization header holding a bare scheme word ("Bearer" with no token) // must not censor answers either. const authorizationTokens = headerEntries .filter(([name]) => name.toLowerCase() === "authorization") .map(([, value]) => value.replace(/^\S+\s+/, "")) .filter((value) => value.length >= CREDENTIAL_MIN_LENGTH); const otherCredentials = [ ...authorizationTokens, // Other headers and env values must look like secrets, not labels: a // short business value (X-Env: prod) must not censor normal answers. ...headerEntries .filter(([name]) => name.toLowerCase() !== "authorization") .map(([, value]) => value) .filter((value) => value.length >= CREDENTIAL_MIN_LENGTH), ...Object.values(auth.env ?? {}).filter((value) => value && value.length >= CREDENTIAL_MIN_LENGTH), ]; if (otherCredentials.some((value) => value && raw.includes(value))) { return "a configured credential"; } } /** Never echo an upstream message that contains any resolved credential value. */ function safeUpstreamMessage(raw: string | undefined, auth: VisionAuth): string { if (!raw) return "no detail returned"; const echo = credentialEcho(raw, auth); if (echo) return `detail withheld: the upstream response echoed ${echo}`; return raw.length > 500 ? `${raw.slice(0, 500)}…` : raw; } /** Ordered thinking levels; a model's thinkingLevelMap maps each to a vendor name or null (= unsupported). */ const THINKING_LEVELS = ["minimal", "low", "medium", "high", "xhigh", "max"] as const; type ReasoningEffort = (typeof THINKING_LEVELS)[number]; /** * Lowest thinking level the model's thinkingLevelMap admits (null = unsupported), * or undefined when it declares no map. Always-thinking models reject "thinking * disabled", which is what pi-ai sends for zai-style APIs when no effort is * given — so defer to the map, the single source of truth the catalog maintains. */ function lowestAdmittedEffort(model: Model): ReasoningEffort | undefined { for (const level of THINKING_LEVELS) { if (typeof model.thinkingLevelMap?.[level] === "string") return level; } return undefined; } function answerText(message: AssistantMessage): string { return message.content .filter((c): c is { type: "text"; text: string } => c.type === "text") .map((c) => c.text) .join("\n") .trim(); } export async function analyzeImages( images: ImageContent[], question: string, ctx: ExtensionContext, signal: AbortSignal | undefined, completeFn: CompleteFn, ): Promise { if (images.length === 0) throw new Error("No images given."); if (!question.trim()) throw new Error("No question given."); const resolved = await resolveVisionModel(ctx); if (!resolved.ok) throw new Error(resolved.error); const modelRef = `${resolved.model.provider}/${resolved.model.id}`; let response: AssistantMessage; try { response = await completeFn( resolved.model, { messages: [ { role: "user", content: [...images, { type: "text", text: question }], timestamp: Date.now(), }, ], }, { apiKey: resolved.auth.apiKey, headers: resolved.auth.headers, env: resolved.auth.env, signal, cacheRetention: "none", // pi-ai's zai/qwen thinking formats send "thinking disabled" when no effort is // given, which always-thinking models (e.g. glm-5.3-flash: "该模型始终思考, 不支持 // 关闭思考") reject. Single rule: pass the lowest level the model's // thinkingLevelMap admits; models without a map get none, and pi-ai only // consumes the option in its thinking branches anyway. reasoningEffort: lowestAdmittedEffort(resolved.model), }, ); } catch (error) { throw new Error( `Vision model ${modelRef} request failed: ${safeUpstreamMessage((error as Error).message, resolved.auth)}`, ); } if (response.stopReason === "error") { throw new Error( `Vision model ${modelRef} returned an error: ${safeUpstreamMessage(response.errorMessage, resolved.auth)}`, ); } const text = answerText(response); if (!text) { throw new Error(`Vision model ${modelRef} returned no text (stopReason: ${response.stopReason}).`); } const echo = credentialEcho(text, resolved.auth); if (echo) { throw new Error(`Vision model ${modelRef} response echoed ${echo}; text withheld.`); } return { text, model: modelRef, imageCount: images.length, usage: response.usage }; } export async function runImageAsk( params: { paths: string[]; question: string }, ctx: ExtensionContext, signal: AbortSignal | undefined, completeFn: CompleteFn, ): Promise> { if (params.paths.length === 0) throw new Error("No image paths given."); if (!params.question.trim()) throw new Error("No question given."); if (ctx.model?.input.includes("image")) { throw new Error("image_ask is unavailable because the current model already supports images."); } // Same budget as the auto path (BYTE-6 A3): unbounded paths meant one model // call could carry dozens of images — a giant request and a huge base64 // blob held in memory. if (params.paths.length > MAX_ATTACHMENT_COUNT) { throw new Error( `Ask about at most ${MAX_ATTACHMENT_COUNT} images per call (got ${params.paths.length}). Group the most relevant ones.`, ); } const images: ImageContent[] = []; let totalBytes = 0; for (const path of params.paths) { const image = await readImageFile(path, ctx.cwd); const byteLength = decodedBase64Length(image.data) ?? 0; totalBytes += byteLength; if (totalBytes > MAX_IMAGE_BYTES) { throw new Error(`${MAX_IMAGE_BYTES / 1024 / 1024}MB total image limit exceeded; use fewer or smaller images.`); } images.push(image); } const analysis = await analyzeImages(images, params.question, ctx, signal, completeFn); return { content: [{ type: "text", text: analysis.text }], details: { model: analysis.model, imageCount: analysis.imageCount }, }; } export function registerImageAskTool(pi: ExtensionAPI, completeFn: CompleteFn = complete): void { pi.registerTool({ name: TOOL_NAME, label: "Image Ask", description: "Ask a vision model about local images and return a text answer.", promptSnippet: "Ask a vision model about local images.", promptGuidelines: [ `${TOOL_NAME}: use when work depends on image contents; never infer contents from filenames.`, ], parameters, async execute(_toolCallId, params, signal, _onUpdate, ctx) { return runImageAsk(params, ctx, signal, completeFn); }, }); }