/** * Narration QA — Gemini-Vision slide/narration alignment (the `check_slide` * port deferred from `qa-narration.ts`). * * Ported from `skills/video-replicator/scripts/bunty_narration_check.py` * `check_slide()`: for each slide N, ask Gemini Vision whether the planned * narration for scene N actually describes what slide N's image shows. Three * verdicts per slide (plus `error` for API/parse failures, mirroring the * Python): * * aligned — narration directly describes the slide * partial — right ballpark, but misses or invents a key element * mismatch — narration is about a different beat entirely * * Run after drafting narration, BEFORE generating TTS. `ok` is false only when * a `mismatch` is present (the Python's exit contract); `partial`/`error` are * advisory. The local, deterministic structural checks live in the pure * sibling `qa-narration.ts` — this module is the network half, kept separate * so the sibling stays unit-testable without I/O. * * Transport (key pool / endpoint override / injectable fetcher / the two-line * `verdict:` / `reason:` reply contract) is the shared * {@link classifyImageWithGemini} in `gemini-vision-classify.ts`, used by both * assemble QA vision modules. */ import { classifyImageWithGemini, parseTwoLineVerdict, resolveVisionQaEndpoint, DEFAULT_GEMINI_VISION_QA_ENDPOINT, } from './gemini-vision-classify.js'; import { VclawError } from '../errors.js'; export const DEFAULT_QA_NARRATION_VISION_ENDPOINT = DEFAULT_GEMINI_VISION_QA_ENDPOINT; const ALIGNMENT_VERDICTS = ['aligned', 'partial', 'mismatch'] as const; export type SlideAlignmentVerdict = 'aligned' | 'partial' | 'mismatch' | 'error'; export interface SlideAlignmentSlide { sceneIndex: number; /** Path to the slide image shown while this scene's narration plays. */ imagePath: string; /** Planned narration text for this scene. */ narration: string; } export interface CheckSlideAlignmentInput { slides: SlideAlignmentSlide[]; /** Override the Gemini endpoint (else VCLAW_GEMINI_API_ENDPOINT, else default). */ endpoint?: string; /** Explicit API key — bypasses the env-backed key pool. */ keyOverride?: string; /** Injectable fetch for offline tests. */ fetcher?: typeof fetch; } export interface SlideAlignmentResult { sceneIndex: number; verdict: SlideAlignmentVerdict; reason: string; } export interface CheckSlideAlignmentResult { results: SlideAlignmentResult[]; /** False only when at least one slide is a `mismatch` (errors are advisory). */ ok: boolean; } /** * The audit prompt, generalized from the Python original (which was worded for * its cricket-recap pipeline). The reply contract is unchanged: an EXACT * two-line `verdict:` / `reason:` reply parsed by {@link parseSlideAlignmentReply}. */ export function buildSlideAlignmentPrompt(sceneIndex: number, narration: string): string { return [ 'You are auditing a narrated slide-deck video pipeline. Below is one slide image from the deck and the planned narration that will play while this slide is on screen. Your job: classify whether the narration matches what the slide actually shows.', '', `Slide ${sceneIndex} narration:`, `"${narration}"`, '', 'Classify as one of:', '- aligned: narration directly describes what the slide shows (key facts, names, beat). Minor flourish/catchphrase wording is fine.', '- partial: narration is in the right ballpark but misses a key element on the slide OR mentions something not shown.', '- mismatch: narration describes a completely different beat than the slide shows.', '', 'Reply in this EXACT format (no other text):', 'verdict: ', 'reason: ', ].join('\n'); } /** * Parse the two-line `verdict:` / `reason:` reply. An unrecognized verdict * value falls back to `error` with the full reply text as the reason, exactly * like the Python original — so a refusal or free-form answer is surfaced * verbatim instead of being misclassified. */ export function parseSlideAlignmentReply(text: string): { verdict: SlideAlignmentVerdict; reason: string } { return parseTwoLineVerdict(text, ALIGNMENT_VERDICTS); } /** * Classify slide/narration alignment for every slide, sequentially (one Gemini * call per slide, matching the Python; slide decks are small and sequential * keeps key-pool cooldown behaviour simple). Per-slide failures (missing * image, HTTP error, unparseable reply) become `error` results rather than * aborting the deck. Throws only on invalid input. */ export async function checkSlideAlignment( input: CheckSlideAlignmentInput, ): Promise { if (!input || !Array.isArray(input.slides)) { throw new VclawError( 'unexpected_internal_error', 'checkSlideAlignment: input.slides must be an array', ); } const endpoint = resolveVisionQaEndpoint(input.endpoint); const results: SlideAlignmentResult[] = []; for (const slide of input.slides) { const classified = await classifyImageWithGemini({ imagePath: slide.imagePath, prompt: buildSlideAlignmentPrompt(slide.sceneIndex, slide.narration), allowedVerdicts: ALIGNMENT_VERDICTS, endpoint, keyOverride: input.keyOverride, fetcher: input.fetcher, }); results.push({ sceneIndex: slide.sceneIndex, ...classified }); } return { results, ok: results.every((r) => r.verdict !== 'mismatch') }; }