import { createHash } from "node:crypto"; import { StringEnum, uuidv7, type ImageContent, type TextContent, type Tool, type Usage, } from "@earendil-works/pi-ai"; import type { BeforeAgentStartEvent, ExtensionAPI, ExtensionContext, ToolResultEvent, } from "@earendil-works/pi-coding-agent"; import { type Static, Type } from "typebox"; import { Value } from "typebox/value"; import { clearFocusedReadStatus, registerFocusedRead, } from "./focused-read.ts"; const STATUS_KEY = "focused-vision"; const OBSERVATION_OPEN = "; const VISION_REPORT_TOOL: Tool = { name: "report_visual_observation", description: "Return the complete visual conclusion, visible text, spatial summary, overall clarity, and every task-relevant uncertain region. Always call exactly once.", parameters: VisualReportSchema, constrainedSampling: { type: "json_schema", strict: "prefer" }, }; type VisionReasoning = "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; type VisionContent = TextContent | ImageContent; export interface FocusedVisionConfig { enabled: boolean; routeInputImages: boolean; routeToolResultImages: boolean; overrideRead: boolean; provider: string; model: string; maxImages: number; maxTokens: number; timeoutMs: number; cacheEntries: number; reasoningEffort: VisionReasoning; } /** Backward-compatible type name for existing imports. */ export type VisionRouterConfig = FocusedVisionConfig; interface VisualItem { image: ImageContent; context: string; originalIndex: number; } interface VisualRequest { query: string; source: string; items: VisualItem[]; trailingContext?: string; } interface VisionAnalysis { text: string; report: VisualReport; modelRef: string; selectedImages: number; totalImages: number; cached: boolean; usage?: Usage; } interface CachedVisionAnalysis { text: string; report: VisualReport; modelRef: string; selectedImages: number; totalImages: number; } function envBoolean(value: string | undefined, defaultValue: boolean): boolean { if (value === undefined) return defaultValue; const normalized = value.trim().toLowerCase(); if (["1", "true", "yes", "on"].includes(normalized)) return true; if (["0", "false", "no", "off"].includes(normalized)) return false; return defaultValue; } function envInteger( value: string | undefined, defaultValue: number, minimum: number, maximum: number, ): number { if (value === undefined) return defaultValue; const parsed = Number.parseInt(value, 10); if (!Number.isFinite(parsed)) return defaultValue; return Math.max(minimum, Math.min(maximum, parsed)); } function reasoningEffort(value: string | undefined): VisionReasoning { const normalized = value?.trim().toLowerCase(); if ( normalized === "minimal" || normalized === "low" || normalized === "medium" || normalized === "high" || normalized === "xhigh" || normalized === "max" ) { return normalized; } return "low"; } function envSetting( env: NodeJS.ProcessEnv, ...names: string[] ): string | undefined { for (const name of names) { if (env[name] !== undefined) return env[name]; } return undefined; } export function loadFocusedVisionConfig( env: NodeJS.ProcessEnv = process.env, ): FocusedVisionConfig { return { enabled: !envBoolean( envSetting( env, "PI_TEXT_MODEL_VISION_DISABLED", "PI_FOCUSED_VISION_DISABLED", "PI_VISION_ROUTER_DISABLED", ), false, ), routeInputImages: envBoolean( envSetting( env, "PI_TEXT_MODEL_VISION_INPUT", "PI_FOCUSED_VISION_INPUT", "PI_VISION_ROUTER_INPUT", ), true, ), routeToolResultImages: envBoolean( envSetting( env, "PI_TEXT_MODEL_VISION_TOOL_RESULTS", "PI_FOCUSED_VISION_TOOL_RESULTS", "PI_VISION_ROUTER_TOOL_RESULTS", ), true, ), overrideRead: envBoolean( envSetting( env, "PI_TEXT_MODEL_VISION_READ_OVERRIDE", "PI_FOCUSED_VISION_READ_OVERRIDE", ), true, ), provider: DEFAULT_PROVIDER, model: DEFAULT_MODEL, maxImages: envInteger( envSetting( env, "PI_TEXT_MODEL_VISION_MAX_IMAGES", "PI_FOCUSED_VISION_MAX_IMAGES", "PI_VISION_ROUTER_MAX_IMAGES", ), DEFAULT_MAX_IMAGES, 1, 32, ), maxTokens: envInteger( envSetting( env, "PI_TEXT_MODEL_VISION_MAX_TOKENS", "PI_FOCUSED_VISION_MAX_TOKENS", "PI_VISION_ROUTER_MAX_TOKENS", ), DEFAULT_MAX_TOKENS, 256, 16_384, ), timeoutMs: envInteger( envSetting( env, "PI_TEXT_MODEL_VISION_TIMEOUT_MS", "PI_FOCUSED_VISION_TIMEOUT_MS", "PI_VISION_ROUTER_TIMEOUT_MS", ), DEFAULT_TIMEOUT_MS, 1_000, 600_000, ), cacheEntries: envInteger( envSetting( env, "PI_TEXT_MODEL_VISION_CACHE_ENTRIES", "PI_FOCUSED_VISION_CACHE_ENTRIES", "PI_VISION_ROUTER_CACHE_ENTRIES", ), DEFAULT_CACHE_ENTRIES, 0, 128, ), reasoningEffort: reasoningEffort( envSetting( env, "PI_TEXT_MODEL_VISION_REASONING_EFFORT", "PI_FOCUSED_VISION_REASONING_EFFORT", "PI_VISION_ROUTER_REASONING_EFFORT", ), ), }; } /** Backward-compatible export for existing imports. */ export const loadVisionRouterConfig = loadFocusedVisionConfig; function truncate(value: string, maximum: number): string { if (value.length <= maximum) return value; return `${value.slice(0, Math.max(0, maximum - 16))}\n[… truncated …]`; } function stripKnownImageOmissionNotes(value: string): string { return value .replace( /\[Current model does not support images\. The image will be omitted from this request\.\]/g, "", ) .replace(/\((?:tool )?image omitted: model does not support images\)/gi, "") .trim(); } function withoutImageOmissionNotes( content: readonly VisionContent[], ): VisionContent[] { const cleaned: VisionContent[] = []; for (const block of content) { if (block.type === "image") { cleaned.push(block); continue; } const text = stripKnownImageOmissionNotes(block.text); if (text) cleaned.push({ type: "text", text }); } return cleaned; } function stripVisionObservations(value: string): string { return value .replace( /<(?:focused_vision|vision)_observation\b[^>]*>[\s\S]*?<\/(?:focused_vision|vision)_observation>/g, "", ) .trim(); } function contentText(content: unknown): string { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; return content .filter( (part): part is TextContent => Boolean(part) && typeof part === "object" && (part as { type?: unknown }).type === "text" && typeof (part as { text?: unknown }).text === "string", ) .map((part) => part.text) .join("\n"); } export function latestUserText(ctx: ExtensionContext): string { let entries: ReturnType; try { entries = ctx.sessionManager.getBranch(); } catch { return ""; } for (let index = entries.length - 1; index >= 0; index--) { const entry = entries[index]; if (entry?.type !== "message" || entry.message.role !== "user") continue; const text = stripVisionObservations(contentText(entry.message.content)); if (text) return truncate(text, MAX_USER_PROMPT_CHARS); } return ""; } export function visualItemsFromContent(content: readonly VisionContent[]): { items: VisualItem[]; trailingContext: string; } { const items: VisualItem[] = []; let pendingText: string[] = []; let imageIndex = 0; for (const block of content) { if (block.type === "text") { const cleaned = stripKnownImageOmissionNotes(block.text); if (cleaned) pendingText.push(cleaned); continue; } items.push({ image: block, context: truncate(pendingText.join("\n"), MAX_ITEM_CONTEXT_CHARS), originalIndex: imageIndex, }); imageIndex++; pendingText = []; } return { items, trailingContext: truncate( pendingText.join("\n"), MAX_TRAILING_CONTEXT_CHARS, ), }; } export function selectEvenly(values: readonly T[], maximum: number): Array<{ value: T; index: number; }> { if (values.length <= maximum) { return values.map((value, index) => ({ value, index })); } if (maximum <= 1) { const index = Math.floor((values.length - 1) / 2); return [{ value: values[index]!, index }]; } const selected: Array<{ value: T; index: number }> = []; const seen = new Set(); for (let position = 0; position < maximum; position++) { const index = Math.round( (position * (values.length - 1)) / (maximum - 1), ); if (seen.has(index)) continue; seen.add(index); selected.push({ value: values[index]!, index }); } return selected; } function requestContent( request: VisualRequest, maxImages: number, ): { content: VisionContent[]; selectedItems: VisualItem[]; } { const selected = selectEvenly(request.items, maxImages).map( ({ value }) => value, ); const content: VisionContent[] = [ { type: "text", text: [ `视觉来源:${request.source}`, `用户问题:${truncate(request.query || "请完整描述视觉内容。", MAX_USER_PROMPT_CHARS)}`, request.items.length > selected.length ? `共有 ${request.items.length} 个视觉项;为控制开销,已均匀选取 ${selected.length} 个。` : `共有 ${request.items.length} 个视觉项。`, "请按视觉项编号给出观察,并优先回答用户问题。", ].join("\n"), }, ]; for (const item of selected) { const label = [`[视觉项 ${item.originalIndex + 1}/${request.items.length}]`]; if (item.context) label.push(`关联上下文:\n${item.context}`); content.push({ type: "text", text: label.join("\n") }); content.push(item.image); } if (request.trailingContext) { content.push({ type: "text", text: `尾随上下文:\n${request.trailingContext}`, }); } return { content, selectedItems: selected }; } function cacheKey( config: VisionRouterConfig, request: VisualRequest, selectedItems: readonly VisualItem[], ): string { const hash = createHash("sha256"); hash.update(config.provider); hash.update("\0"); hash.update(config.model); hash.update("\0"); hash.update(request.source); hash.update("\0"); hash.update(request.query); hash.update("\0"); hash.update(request.trailingContext ?? ""); for (const item of selectedItems) { hash.update("\0"); hash.update(String(item.originalIndex)); hash.update("\0"); hash.update(item.context); hash.update("\0"); hash.update(item.image.mimeType); hash.update("\0"); hash.update(item.image.data); } return hash.digest("hex"); } function safeErrorMessage(error: unknown): string { const raw = error instanceof Error ? error.message : String(error); return truncate( raw .replace(/\bBearer\s+\S+/gi, "Bearer [redacted]") .replace(/\bsk-[A-Za-z0-9_-]{8,}\b/g, "sk-[redacted]"), MAX_ERROR_CHARS, ); } function validateVisualReportInvariants(report: VisualReport): void { for (const region of report.uncertainRegions) { const [x1, y1, x2, y2] = region.bbox; if (x1 >= x2 || y1 >= y2) { throw new Error( `结构化视觉报告包含无效 bbox:${JSON.stringify(region.bbox)}`, ); } } if (report.clarity === "clear" && report.uncertainRegions.length > 0) { throw new Error("clarity=clear 时 uncertainRegions 必须为空"); } if (report.clarity !== "clear" && report.uncertainRegions.length === 0) { throw new Error(`${report.clarity} 报告必须包含 uncertainRegions`); } const relevantUncertainty = report.uncertainRegions.some( (region) => region.affectsAnswer, ); if (report.focusRequired !== relevantUncertainty) { throw new Error( "focusRequired 必须与是否存在 affectsAnswer=true 的模糊区域一致", ); } } export function parseVisualReport(content: readonly unknown[]): VisualReport { const toolCalls = content.filter( (block): block is { type: "toolCall"; name: string; arguments: Record; } => Boolean(block) && typeof block === "object" && (block as { type?: unknown }).type === "toolCall" && typeof (block as { name?: unknown }).name === "string" && Boolean((block as { arguments?: unknown }).arguments) && typeof (block as { arguments?: unknown }).arguments === "object", ); if (toolCalls.length !== 1) { throw new Error( `视觉模型必须调用一次 ${VISION_REPORT_TOOL.name},实际为 ${toolCalls.length} 次`, ); } const call = toolCalls[0]!; if (call.name !== VISION_REPORT_TOOL.name) { throw new Error(`视觉模型调用了意外工具:${call.name}`); } if (!Value.Check(VisualReportSchema, call.arguments)) { throw new Error("视觉模型返回的结构化报告不符合 VisualReport schema"); } const report = call.arguments as VisualReport; validateVisualReportInvariants(report); return report; } export function formatVisualReport(report: VisualReport): string { return JSON.stringify(report, null, 2); } function escapeObservationBoundary(value: string): string { return value .replaceAll(OBSERVATION_CLOSE, "<\\/focused_vision_observation>") .replaceAll("", "<\\/vision_observation>"); } export function formatVisionObservation(analysis: VisionAnalysis): string { const attributes = [ `model="${analysis.modelRef.replaceAll('"', """)}"`, `images="${analysis.selectedImages}/${analysis.totalImages}"`, `clarity="${analysis.report.clarity}"`, `focus_required="${analysis.report.focusRequired}"`, `uncertain_regions="${analysis.report.uncertainRegions.length}"`, ]; if (analysis.cached) attributes.push('cached="true"'); return `${OBSERVATION_OPEN} ${attributes.join(" ")}> ${escapeObservationBoundary(analysis.text)} ${OBSERVATION_CLOSE}`; } export function combineUsage( first: Usage | undefined, second: Usage | undefined, ): Usage | undefined { if (!first) return second; if (!second) return first; const reasoning = first.reasoning === undefined && second.reasoning === undefined ? undefined : (first.reasoning ?? 0) + (second.reasoning ?? 0); return { input: first.input + second.input, output: first.output + second.output, cacheRead: first.cacheRead + second.cacheRead, cacheWrite: first.cacheWrite + second.cacheWrite, ...(reasoning === undefined ? {} : { reasoning }), totalTokens: first.totalTokens + second.totalTokens, cost: { input: first.cost.input + second.cost.input, output: first.cost.output + second.cost.output, cacheRead: first.cost.cacheRead + second.cost.cacheRead, cacheWrite: first.cost.cacheWrite + second.cost.cacheWrite, total: first.cost.total + second.cost.total, }, }; } function sourceFromToolResult(event: ToolResultEvent): string { const possiblePath = typeof event.input.path === "string" ? event.input.path : typeof event.input.image_path === "string" ? event.input.image_path : typeof event.input.file_path === "string" ? event.input.file_path : typeof event.input.video_path === "string" ? event.input.video_path : undefined; return truncate( possiblePath ? `工具 ${event.toolName} 的输出(${possiblePath})` : `工具 ${event.toolName} 的输出`, 1_000, ); } function mainModelNeedsVision(ctx: ExtensionContext): boolean { return Boolean(ctx.model && !ctx.model.input.includes("image")); } function visualSidecarIssue( ctx: ExtensionContext, config: FocusedVisionConfig, ): string | undefined { const model = ctx.modelRegistry.find(config.provider, config.model); if (!model) { return `需要先在 Pi 中配置 ${config.provider}/${config.model};可用 pi --list-models ${config.provider} 检查`; } if (!model.input.includes("image")) { return `${config.provider}/${config.model} 未声明 image 输入能力`; } if (!ctx.modelRegistry.hasConfiguredAuth(model)) { return `${config.provider} 尚未完成认证配置`; } return undefined; } class VisionRouterRuntime { private readonly cache = new Map(); private readonly pending = new Map>(); private readonly reportedErrors = new Set(); private activeCalls = 0; constructor(private readonly config: VisionRouterConfig) {} reset(): void { this.cache.clear(); this.pending.clear(); this.reportedErrors.clear(); this.activeCalls = 0; } stats(): { cacheEntries: number; pending: number; activeCalls: number } { return { cacheEntries: this.cache.size, pending: this.pending.size, activeCalls: this.activeCalls, }; } reportError(ctx: ExtensionContext, message: string): void { if (this.reportedErrors.has(message)) return; this.reportedErrors.add(message); if (ctx.hasUI) ctx.ui.notify(`Text-model vision: ${message}`, "warning"); } private setWorking(ctx: ExtensionContext, active: boolean): void { this.activeCalls += active ? 1 : -1; this.activeCalls = Math.max(0, this.activeCalls); if (!ctx.hasUI) return; ctx.ui.setStatus( STATUS_KEY, this.activeCalls > 0 ? `focus: ${this.config.model}${this.activeCalls > 1 ? ` ×${this.activeCalls}` : ""}` : undefined, ); } private cacheGet(key: string): CachedVisionAnalysis | undefined { const value = this.cache.get(key); if (!value) return undefined; this.cache.delete(key); this.cache.set(key, value); return value; } private cacheSet(key: string, value: CachedVisionAnalysis): void { if (this.config.cacheEntries <= 0) return; this.cache.delete(key); this.cache.set(key, value); while (this.cache.size > this.config.cacheEntries) { const oldest = this.cache.keys().next().value as string | undefined; if (oldest === undefined) break; this.cache.delete(oldest); } } async analyze( ctx: ExtensionContext, request: VisualRequest, ): Promise { const { content, selectedItems } = requestContent( request, this.config.maxImages, ); const key = cacheKey(this.config, request, selectedItems); const cached = this.cacheGet(key); if (cached) return { ...cached, cached: true }; const alreadyPending = this.pending.get(key); if (alreadyPending) { const result = await alreadyPending; return { ...result, cached: true, usage: undefined }; } const operation = this.performAnalysis(ctx, request, content, selectedItems); this.pending.set(key, operation); try { const result = await operation; this.cacheSet(key, { text: result.text, report: result.report, modelRef: result.modelRef, selectedImages: result.selectedImages, totalImages: result.totalImages, }); return result; } finally { this.pending.delete(key); } } private async performAnalysis( ctx: ExtensionContext, request: VisualRequest, content: VisionContent[], selectedItems: VisualItem[], ): Promise { const model = ctx.modelRegistry.find( this.config.provider, this.config.model, ); if (!model) { throw new Error( `找不到视觉模型 ${this.config.provider}/${this.config.model}`, ); } if (!model.input.includes("image")) { throw new Error( `模型 ${this.config.provider}/${this.config.model} 未声明 image 输入能力`, ); } if (!ctx.modelRegistry.hasConfiguredAuth(model)) { throw new Error(`未配置 ${this.config.provider} 的认证信息`); } this.setWorking(ctx, true); try { let combinedUsage: Usage | undefined; let previousValidationError: string | undefined; for (let attempt = 0; attempt < 2; attempt++) { const systemPrompt = previousValidationError ? `${VISION_SYSTEM_PROMPT}\n\n上一次结构化返回无效:${previousValidationError}\n这次只调用一次 ${VISION_REPORT_TOOL.name},并严格满足 schema 与字段一致性。` : VISION_SYSTEM_PROMPT; const response = await ctx.modelRegistry.complete( model, { systemPrompt, messages: [ { role: "user", content, timestamp: Date.now(), }, ], tools: [VISION_REPORT_TOOL], }, { maxTokens: this.config.maxTokens, reasoningEffort: this.config.reasoningEffort, timeoutMs: this.config.timeoutMs, signal: ctx.signal, cacheRetention: "none", sessionId: uuidv7(), toolChoice: { type: "function", function: { name: VISION_REPORT_TOOL.name }, }, }, ); combinedUsage = combineUsage(combinedUsage, response.usage); if (response.stopReason === "error") { throw new Error(response.errorMessage || "视觉模型调用失败"); } if (response.stopReason === "aborted") { throw new Error("视觉模型调用已取消"); } try { const report = parseVisualReport(response.content); return { text: formatVisualReport(report), report, modelRef: `${model.provider}/${model.id}`, selectedImages: selectedItems.length, totalImages: request.items.length, cached: false, usage: combinedUsage, }; } catch (error) { previousValidationError = safeErrorMessage(error); if (attempt === 1) { throw new Error( `视觉模型连续两次未返回有效结构化报告:${previousValidationError}`, ); } } } throw new Error("视觉模型没有返回结构化报告"); } finally { this.setWorking(ctx, false); } } } function inputRequest(event: BeforeAgentStartEvent): VisualRequest { return { query: truncate(event.prompt.trim(), MAX_USER_PROMPT_CHARS), source: "用户直接附加的图片", items: (event.images ?? []).map((image, originalIndex) => ({ image, context: "", originalIndex, })), }; } function toolResultRequest( event: ToolResultEvent, ctx: ExtensionContext, ): VisualRequest { const { items, trailingContext } = visualItemsFromContent(event.content); return { query: latestUserText(ctx) || "请完整描述这些视觉输出,并转写所有可见文字。", source: sourceFromToolResult(event), items, trailingContext, }; } export function registerFocusedVision( pi: ExtensionAPI, config: FocusedVisionConfig = loadFocusedVisionConfig(), ): void { if (!config.enabled) return; const runtime = new VisionRouterRuntime(config); const focusedRead = registerFocusedRead(pi, { enabled: config.overrideRead, }); pi.on("session_start", async (_event, ctx) => { runtime.reset(); await focusedRead.dispose(); if (ctx.hasUI) ctx.ui.setStatus(STATUS_KEY, undefined); clearFocusedReadStatus(ctx); if (ctx.hasUI && mainModelNeedsVision(ctx)) { const issue = visualSidecarIssue(ctx, config); if (issue) ctx.ui.notify(`Text-model vision: ${issue}`, "warning"); } }); pi.on("session_shutdown", async (_event, ctx) => { runtime.reset(); await focusedRead.dispose(); if (ctx.hasUI) ctx.ui.setStatus(STATUS_KEY, undefined); clearFocusedReadStatus(ctx); }); if (config.routeInputImages) { pi.on("before_agent_start", async (event, ctx) => { if (!event.images?.length || !mainModelNeedsVision(ctx)) return; try { const analysis = await runtime.analyze(ctx, inputRequest(event)); return { message: { customType: "focused-vision-observation", content: formatVisionObservation(analysis), display: false, details: { model: analysis.modelRef, selectedImages: analysis.selectedImages, totalImages: analysis.totalImages, cached: analysis.cached, clarity: analysis.report.clarity, focusRequired: analysis.report.focusRequired, uncertainRegions: analysis.report.uncertainRegions, usage: analysis.usage, }, }, }; } catch (error) { const message = safeErrorMessage(error); runtime.reportError(ctx, message); return { message: { customType: "focused-vision-observation", content: `[聚焦视觉失败:${message}]`, display: false, details: { error: message }, }, }; } }); } if (config.routeToolResultImages) { pi.on("tool_result", async (event, ctx) => { if ( event.isError || !event.content.some((block) => block.type === "image") || !mainModelNeedsVision(ctx) ) { return; } const cleanContent = withoutImageOmissionNotes(event.content); try { const analysis = await runtime.analyze( ctx, toolResultRequest(event, ctx), ); return { content: [ ...cleanContent, { type: "text" as const, text: formatVisionObservation(analysis) }, ], usage: combineUsage(event.usage, analysis.usage), }; } catch (error) { const message = safeErrorMessage(error); runtime.reportError(ctx, message); return { content: [ ...cleanContent, { type: "text" as const, text: `[聚焦视觉失败:${message}]` }, ], }; } }); } } /** Backward-compatible export for code that imported the former router. */ export const registerVisionRouter = registerFocusedVision; export default registerFocusedVision;