/** * Vision LLM calls for the browser agent. * * Multi-stage JSON action extraction, the streaming chat-completion client * with its watchdogs, the screenshot analyzer (fast/deep attempts) and the * redundancy check. * * Extracted from src/cli/commands/browse.ts (line budget) - pure code * motion, no behaviour change. */ import { type VisionFamily } from '../llm/models'; import { type AgentAction, type CdpPage, type ElementInfo, type LLMResult, type QuestionFingerprint, type VisionHarness } from './browse-types'; export interface RedundancyResult { decision: 'skip' | 'partial' | 'full'; reason: string; matchPercent?: number; } /** Ask the LLM to decide if an action is redundant — is the current code already correct? */ export declare function isActionRedundant(baseUrl: string, apiKey: string, model: string, page: CdpPage, elements: ElementInfo[], action: AgentAction, timeoutMs: number, family?: VisionFamily, /** Opt-in harness backend (one CLI turn); absent = direct call (unchanged). */ harness?: VisionHarness | null, /** Run-stable OpenCode session id (sent as x-opencode-session on Zen/Go). */ sessionId?: string | null): Promise; /** * Multi-stage JSON extraction pipeline. * Handles: markdown fences, explanations around JSON, malformed escaping, * trailing commas, unquoted keys, and scattered action objects. */ export declare function extractJsonActions(content: string): AgentAction[]; /** Try JSON.parse, return array or null. */ export declare function tryParseJson(str: string): AgentAction[] | null; /** Validate that each action has required fields. */ export declare function validateActions(actions: any[]): boolean; /** * Extract the outermost [ ... ] using bracket counting. * Handles nested arrays/objects without greedy matching. */ export declare function extractBalancedArray(str: string): string | null; /** * Fix common LLM escaping errors in JSON strings. * Handles: double-escaped newlines, unescaped control chars, trailing commas. */ export declare function fixLlmEscaping(str: string): string; /** * Last resort: extract individual { "action": ... } objects scattered in text. */ export declare function extractScatteredActions(str: string): AgentAction[]; /** True when the text contains a JSON array (the only shape actions take). */ export declare function looksLikeJsonActions(text: string): boolean; /** * Extract the model text from a non-streaming chat-completions JSON body. * * `reasoning_content` is only used as a fallback when it actually contains a * JSON array - a reasoning trace (prose) must NEVER be handed to the action * parser, or a normal answer is misreported as a parse failure. */ export declare function extractModelText(json: any): string; export interface StreamOutcome { kind: 'ok' | 'http' | 'stalled'; content?: string; status?: number; errorText?: string; /** Which watchdog tripped ('first-token' = nothing streamed at all). */ phase?: 'first-token' | 'idle'; elapsedMs?: number; } /** One vision turn, expressed per wire family. */ export interface VisionPrompt { system: string; userText: string; imageBase64?: string; imageMime?: string; maxTokens: number; /** `chat` only — other families are always non-streaming. */ stream?: boolean; } /** * Build the request body for one vision turn in the family's own shape. * `chat` is the OpenAI-compatible `chat/completions` schema; `responses` * follows the OpenAI Responses API (`instructions` + `input` blocks); * `messages` follows the Anthropic Messages API; `gemini` follows * `generateContent` (`system_instruction` + `contents` parts). */ export declare function buildVisionBody(family: VisionFamily, model: string, prompt: VisionPrompt): unknown; /** Headers for one vision POST (the `api-key` mirror covers Azure-style gateways). */ export declare function visionHeaders(apiKey: string, family: VisionFamily, stream: boolean, sessionId?: string | null, url?: string): Record; /** * Extract the model text from a non-streaming response body in the family's * own shape. Returns '' when nothing usable is present (callers report the * empty/parse error as before). */ export declare function extractVisionText(json: any, family: VisionFamily): string; /** * POST one non-streaming vision turn (every family except streamed `chat`). * The caller's AbortSignal (screen-change / tab-switch / timeout) still owns * cancellation — only the first-token/idle streaming watchdogs don't apply. */ export declare function postVisionOnce(opts: { url: string; apiKey: string; family: VisionFamily; body: unknown; signal: AbortSignal; /** Run-stable OpenCode session id (sent as x-opencode-session on Zen/Go). */ sessionId?: string | null; }): Promise<{ kind: 'ok' | 'http'; content?: string; status?: number; errorText?: string; }>; /** * POST /chat/completions with `stream: true` and read the SSE body. * * Non-streaming never resolves until the provider finishes the whole answer, so * a long reasoning response looked like a 150s+ "hang" with zero feedback. With * streaming we see the first token, log its latency, and can catch a genuinely * stalled request (no bytes on the wire at all). `: PROCESSING` keepalives are * NOT tokens but DO prove the connection is alive, so they reset the watchdog; * only real wire silence trips it. * * The caller's `signal` (screen-change / overall timeout) still aborts this. */ export declare function streamChatCompletion(opts: { url: string; apiKey: string; body: unknown; signal: AbortSignal; firstTokenTimeoutMs: number; idleTimeoutMs: number; onFirstToken?: (ms: number) => void; /** Run-stable OpenCode session id (sent as x-opencode-session on Zen/Go). */ sessionId?: string | null; }): Promise; /** * Question-type label for button/div/li option widgets. These carry no native * type (and usually no role), so the loop cannot know whether the question * takes one answer or several — and must not guess. The model has the * question text, the option texts with their selection states and the * screenshot, so the single-vs-multiple decision is delegated to it outright * (see the FOR CHOICE QUESTIONS directive in the user turn). */ export declare const CHOICE_WIDGET_LABEL = "choice (single OR multiple \u2014 YOU decide from the question: one correct answer \u2192 click it; several correct \u2192 return one click action per correct option in this ONE response)"; export declare function analyzeScreenshot(baseUrl: string, apiKey: string, model: string, screenshotBase64: string, elements: ElementInfo[], pageContext: string, timeoutMs: number, attempt: number, maxAttempts: number, page?: CdpPage, lastFingerprint?: QuestionFingerprint[] | null, budget?: { maxTokens: number; deep: boolean; }, family?: VisionFamily, /** Opt-in harness backend (one CLI turn per call); absent = direct (unchanged). */ harness?: VisionHarness | null, /** Run-stable OpenCode session id (sent as x-opencode-session on Zen/Go). */ sessionId?: string | null): Promise; /** Like analyzeScreenshot but accepts a custom system prompt (for optimization pass). */ export declare function analyzeScreenshotWithPrompt(baseUrl: string, apiKey: string, model: string, screenshotBase64: string, elements: ElementInfo[], pageContext: string, timeoutMs: number, attempt: number, maxAttempts: number, customSystemPrompt: string, page?: CdpPage, family?: VisionFamily, /** Opt-in harness backend (one CLI turn); absent = direct (unchanged). */ harness?: VisionHarness | null, /** Run-stable OpenCode session id (sent as x-opencode-session on Zen/Go). */ sessionId?: string | null): Promise;