/** * Shared Gemini-Vision single-image classifier used by the assemble QA vision * modules (`qa-narration-vision.ts`, `qa-image-vision.ts`). * * Both legacy Python QA scripts (`bunty_narration_check.py check_slide`, * `bunty_image_filter_check.py check_image`) share the same transport shape: * one image + one prompt per call, temperature 0, and an EXACT two-line * `verdict:` / `reason:` reply. This module is that shared half — the per-QA * prompts and verdict ladders stay in their own modules. * * Transport mirrors `gemini-judge.ts`/`gemini-analyze.ts`: key handling via * {@link fetchGeminiWithPool} (round-robin pool with cooldown, or an explicit * `keyOverride`), endpoint overridable via VCLAW_GEMINI_API_ENDPOINT, and an * injectable `fetcher` so tests run fully offline. */ import { readFile } from 'node:fs/promises'; import { extname } from 'node:path'; import { fetchGeminiWithPool } from '../gemini-key-pool.js'; export const DEFAULT_GEMINI_VISION_QA_ENDPOINT = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent'; /** Resolve the endpoint: explicit option → VCLAW_GEMINI_API_ENDPOINT → default. */ export function resolveVisionQaEndpoint(endpoint?: string): string { return endpoint ?? process.env.VCLAW_GEMINI_API_ENDPOINT ?? DEFAULT_GEMINI_VISION_QA_ENDPOINT; } export function slideImageMimeType(imagePath: string): string { switch (extname(imagePath).toLowerCase()) { case '.jpg': case '.jpeg': return 'image/jpeg'; case '.webp': return 'image/webp'; case '.png': default: return 'image/png'; } } // Mirrors extractGeminiText in gemini-judge.ts: unwrap // candidates[0].content.parts[].text, '' rather than throwing. export function extractGeminiText(payload: unknown): string { const candidates = ( payload as { candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }> } ).candidates; return ( candidates?.[0]?.content?.parts ?.map((part) => part.text ?? '') .join('\n') .trim() ?? '' ); } /** * Parse the two-line `verdict:` / `reason:` reply against an allowed verdict * set. An unrecognized verdict value falls back to `'error'` with the full * reply text as the reason, exactly like the Python originals — so a refusal * or free-form answer is surfaced verbatim instead of being misclassified. */ export function parseTwoLineVerdict( text: string, allowed: readonly T[], ): { verdict: T | 'error'; reason: string } { let verdict: T | 'error' = 'error'; let reason = text.trim(); for (const line of text.split('\n')) { const lower = line.toLowerCase(); if (lower.startsWith('verdict:')) { const value = line.slice(line.indexOf(':') + 1).trim().toLowerCase(); if ((allowed as readonly string[]).includes(value)) { verdict = value as T; } } else if (lower.startsWith('reason:')) { reason = line.slice(line.indexOf(':') + 1).trim(); } } return { verdict, reason: verdict === 'error' ? text.trim() : reason }; } export interface ClassifyImageInput { imagePath: string; prompt: string; allowedVerdicts: readonly T[]; /** Pre-resolved endpoint (see {@link resolveVisionQaEndpoint}). */ endpoint: string; /** Explicit API key — bypasses the env-backed key pool. */ keyOverride?: string; /** Injectable fetch for offline tests. */ fetcher?: typeof fetch; } /** * One Gemini-Vision classification call: read the image, POST prompt + * inlineData, parse the two-line reply. Failures (unreadable image, HTTP * error, network throw, refusal) degrade to `{verdict: 'error', reason}` — * the caller decides whether errors are advisory. Never throws. */ export async function classifyImageWithGemini( input: ClassifyImageInput, ): Promise<{ verdict: T | 'error'; reason: string }> { let imageB64: string; try { imageB64 = (await readFile(input.imagePath)).toString('base64'); } catch (error) { return { verdict: 'error', reason: `Could not read slide image ${input.imagePath}: ${(error as Error).message}`, }; } const body = { contents: [ { parts: [ { text: input.prompt }, { inlineData: { mimeType: slideImageMimeType(input.imagePath), data: imageB64 } }, ], }, ], // The Python originals used maxOutputTokens: 200 (tuned for the // non-thinking gemini-2.0-flash). On gemini-3.5-flash, thinking tokens // count against this budget and 200 truncates the reason line mid-word // (live-observed). 1024 leaves room for thought + the two-line reply. generationConfig: { temperature: 0, maxOutputTokens: 1024 }, }; try { const response = await fetchGeminiWithPool( (key) => `${input.endpoint}?key=${key}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), }, { fetcher: input.fetcher, keyOverride: input.keyOverride }, ); if (!response.ok) { const detail = await response.text().catch(() => ''); return { verdict: 'error', reason: `Gemini returned HTTP ${response.status}${detail ? `: ${detail.slice(0, 160)}` : ''}`, }; } const text = extractGeminiText(await response.json()); return parseTwoLineVerdict(text, input.allowedVerdicts); } catch (error) { return { verdict: 'error', reason: `Gemini call failed: ${(error as Error).message}`, }; } } export interface ClassifyTwoImagesInput { /** Path to the reference image (posted FIRST). May be absent — then only the frame is posted. */ referencePath?: string; /** Path to the frame image to compare against the reference (posted SECOND). */ framePath: string; prompt: string; /** Pre-resolved endpoint (see {@link resolveVisionQaEndpoint}). */ endpoint: string; /** Explicit API key — bypasses the env-backed key pool. */ keyOverride?: string; /** Injectable fetch for offline tests. */ fetcher?: typeof fetch; } /** * A two-image Gemini-Vision call: post a reference image + a frame image (in * that order) alongside a free-form prompt and return the model's raw reply * text. The sibling of {@link classifyImageWithGemini} for comparisons that need * BOTH images in one call (e.g. "what appeared in the frame that is NOT in the * keyframe reference") and a richer multi-line reply than the fixed two-line * verdict ladder. Reuses the SAME transport — {@link fetchGeminiWithPool} key * pool, endpoint override, injectable fetcher. The reference image is optional; * when absent only the frame is posted (the prompt should be self-describing). * * Returns `{ text }` on success or `{ text: '', error }` on any failure * (unreadable image, HTTP error, network throw). Never throws — the caller * decides whether errors are advisory. */ export async function classifyTwoImagesWithGemini( input: ClassifyTwoImagesInput, ): Promise<{ text: string; error?: string }> { let frameB64: string; try { frameB64 = (await readFile(input.framePath)).toString('base64'); } catch (error) { return { text: '', error: `Could not read frame image ${input.framePath}: ${(error as Error).message}` }; } const parts: Array> = [{ text: input.prompt }]; // Reference image first (only when it is a readable local file; remote URIs // and unreadable paths are silently skipped — the prompt still names them). if (input.referencePath) { try { const refB64 = (await readFile(input.referencePath)).toString('base64'); parts.push({ inlineData: { mimeType: slideImageMimeType(input.referencePath), data: refB64 } }); } catch { // Reference unreadable (e.g. a gobananas:// URI) — proceed with the frame only. } } parts.push({ inlineData: { mimeType: slideImageMimeType(input.framePath), data: frameB64 } }); const body = { contents: [{ parts }], generationConfig: { temperature: 0, maxOutputTokens: 1024 }, }; try { const response = await fetchGeminiWithPool( (key) => `${input.endpoint}?key=${key}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), }, { fetcher: input.fetcher, keyOverride: input.keyOverride }, ); if (!response.ok) { const detail = await response.text().catch(() => ''); return { text: '', error: `Gemini returned HTTP ${response.status}${detail ? `: ${detail.slice(0, 160)}` : ''}`, }; } return { text: extractGeminiText(await response.json()) }; } catch (error) { return { text: '', error: `Gemini call failed: ${(error as Error).message}` }; } }