/** * Automated character-consistency vision audit. * * This is the engine-side forcing function against identity/costume drift: it * catches the failure where a recurring character's wardrobe or face changes * between rendered scenes (the real Satyavan bug — a dhoti that rendered crimson * in some shots and tan/pink in others, plus a beard that appeared in one * scene) BEFORE a render is presented as done, instead of relying on a human * eyeballing every clip. * * Design: * - The CORE ({@link auditProjectConsistency}) is pure + injectable. It reads the * storyboard + registered character profiles, and for each scene that has a * rendered output (`outputs/scene-.mp4`) or a bound keyframe image, extracts * ONE representative mid-frame and asks an INJECTABLE vision client whether * each scene character matches its canonical reference (face/hair AND * costume/colours), listing any differences. Aggregation is deterministic. * - Border detection is a pure helper ({@link detectBorderFromEdgeStats}) over * already-sampled edge-pixel stats, so it is unit-testable WITHOUT decoding a * real image. A thin ffmpeg-backed sampler ({@link sampleEdgePixelStats}) does * the actual pixel read in production; tests inject their own sampler. * - The DEFAULT vision client ({@link createDefaultVisionClient}) reuses the * EXISTING Gemini infrastructure — the shared {@link classifyImageWithGemini} * transport, the env-backed {@link fetchGeminiWithPool} key pool, and the * VCLAW_GEMINI_API_ENDPOINT override — exactly like the assemble QA vision * modules. No new auth path. Tests inject a fake client and never hit the * network. * * Mid-clip drift detection (the second forcing function): * - Two Veo image-to-video failure modes only show up at full playback and slip * past a single mid-frame check: (1) an element that MATERIALIZES mid-clip and * was not in the scene's keyframe (a floating garland, a flame, a veil pulled * over the head, new cloth/petals), and (2) an anatomy/duplication error (a * third hand, an extra limb, a duplicated person) that only appears at an * arbitrary moment. To catch these, every scene with a rendered clip AND a * bound keyframe (the i2v START IMAGE, resolved from the asset-manifest) is * sampled at K evenly-spaced frames and each frame is compared against the * keyframe by an INJECTABLE {@link FrameInspectionClient}. The default client * reuses the SAME Gemini infra via the two-image * {@link classifyTwoImagesWithGemini} call. Aggregation (dedup + frame-time * tagging) is deterministic; tests inject per-frame verdicts and never hit the * network or ffmpeg. */ import { existsSync } from 'node:fs'; import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { isAbsolute, join, resolve } from 'node:path'; import { artifactPathFor } from './artifact-store.js'; import { listCharacterProfiles, type CharacterProfile } from './characters.js'; import { resolveProjectWorkspace, type VideoProjectWorkspace } from './workspace.js'; import { classifyImageWithGemini, classifyTwoImagesWithGemini, resolveVisionQaEndpoint, } from './assemble/gemini-vision-classify.js'; import { resolveFfmpegBin, resolveFfprobeBin, runFfmpeg, ffprobeDuration } from './assemble/ffmpeg.js'; import { resolveWorkspaceRootFromEnv } from './workspace-root.js'; /** Per-character verdict from the vision client for one scene frame. */ export interface ConsistencyVisionAssessment { /** True iff the on-screen character matches the reference face/hair AND costume/colours. */ match: boolean; /** Human-readable differences (empty when matching). */ issues: string[]; /** True when the vision client suspects an unexpected extra figure in the frame. */ extraFigure?: boolean; } /** Input passed to the vision client for a single (character, scene-frame) pair. */ export interface ConsistencyVisionInput { /** Canonical display name of the character being checked. */ characterName: string; /** Canonical reference (the character's GB reference URL / referenceAssets[0]); may be absent. */ referencePath?: string; /** The character's locked visual descriptor. */ description?: string; /** The character's locked costume/wardrobe (the dhoti-colour anchor). */ costume?: string; /** Path to the extracted scene mid-frame to assess. */ framePath: string; /** Scene index (for context / prompts). */ sceneIndex: number; } /** Injectable vision client — the default reuses the Gemini infra; tests inject a fake. */ export interface ConsistencyVisionClient { assess(input: ConsistencyVisionInput): Promise; } /** Per-frame verdict from the frame-inspection client for one sampled clip frame. */ export interface ConsistencyFrameInspection { /** * Things visible in the sampled frame that are NOT present in the keyframe — * a new object/garment that materialized mid-clip (a floating garland, a * flame, a veil pulled over the head, new cloth/petals). Empty when clean. */ appearingElements: string[]; /** * Anatomy/duplication errors in the sampled frame — a person with more than * two arms/hands, an extra/duplicate limb, or a duplicated character. Empty * when clean. */ anatomyIssues: string[]; } /** Input passed to the frame-inspection client for one sampled clip frame. */ export interface ConsistencyFrameInspectionInput { /** The scene's i2v START IMAGE (keyframe) the frame is compared against. */ keyframePath: string; /** Path to the extracted sampled frame. */ framePath: string; /** Scene index (for context / prompts). */ sceneIndex: number; /** Fraction (0–1) of the clip at which this frame was sampled. */ fraction: number; } /** * Injectable frame-inspection client — compares one sampled frame against the * scene keyframe and reports appearing elements + anatomy issues. The default * reuses the Gemini two-image transport; tests inject a fake per-frame verdict. */ export interface ConsistencyFrameInspectionClient { inspect(input: ConsistencyFrameInspectionInput): Promise; } /** Per-character result inside a scene. */ export interface ConsistencyCharacterResult { name: string; match: boolean; issues: string[]; } /** Per-scene audit result. */ export interface ConsistencySceneResult { sceneIndex: number; /** True when a representative frame was actually extracted + assessed. */ frameChecked: boolean; characters: ConsistencyCharacterResult[]; borderDetected: boolean; extraFigureSuspected: boolean; /** * Elements that materialized mid-clip and were NOT in the scene keyframe, * deduped across the sampled frames and each tagged with the approx frame * fraction (e.g. "t≈0.6: floating garland"). Empty when none / no keyframe. */ appearingElements: string[]; /** * Anatomy/duplication errors (third hand, extra limb, duplicated person), * deduped across the sampled frames and each tagged with the approx frame * fraction (e.g. "t≈0.4: third hand on the figure"). Empty when none. */ anatomyIssues: string[]; /** Number of frames actually sampled across the clip for mid-clip inspection. */ framesSampled: number; } /** The full structured audit report (also persisted to artifacts/consistency-audit.json). */ export interface ConsistencyAuditReport { projectSlug: string; /** * False when any checked character mismatches OR any checked frame has a * border OR any scene has appearing-elements / anatomy issues. */ ok: boolean; generatedAt: string; scenes: ConsistencySceneResult[]; /** * Flat, operator-facing list of every problem found (drift + border + * extra-figure + appearing-element + anatomy). */ findings: string[]; } /** Already-sampled per-edge pixel statistics for the deterministic border check. */ export interface EdgePixelStats { /** Mean luma (0–255) of each edge strip. */ topMeanLuma: number; bottomMeanLuma: number; leftMeanLuma: number; rightMeanLuma: number; /** * Uniformity (0–1) of each edge strip — how flat/featureless it is. A true * letterbox/pillarbox margin is both DARK and near-perfectly uniform; a dark * but textured edge (e.g. a night sky) is dark but NOT uniform. */ topUniformity: number; bottomUniformity: number; leftUniformity: number; rightUniformity: number; } /** A single edge strip's stats, for the pure margin test. */ interface EdgeStrip { meanLuma: number; uniformity: number; } // A margin is a dark, flat strip. Tuned conservatively so a normal frame is // never flagged: luma below this (very dark) AND uniformity above this (flat). const MARGIN_MAX_LUMA = 16; const MARGIN_MIN_UNIFORMITY = 0.9; function isMargin(strip: EdgeStrip): boolean { return strip.meanLuma <= MARGIN_MAX_LUMA && strip.uniformity >= MARGIN_MIN_UNIFORMITY; } /** * Pure border/letterbox detector over already-sampled edge stats. A border is * detected when EITHER pair of opposite edges (top+bottom, i.e. letterbox, OR * left+right, i.e. pillarbox) are both dark, near-uniform margins. Requiring a * matched PAIR (not a lone edge) avoids flagging a single dark-but-textured edge * like a night sky. */ export function detectBorderFromEdgeStats(stats: EdgePixelStats): boolean { const top: EdgeStrip = { meanLuma: stats.topMeanLuma, uniformity: stats.topUniformity }; const bottom: EdgeStrip = { meanLuma: stats.bottomMeanLuma, uniformity: stats.bottomUniformity }; const left: EdgeStrip = { meanLuma: stats.leftMeanLuma, uniformity: stats.leftUniformity }; const right: EdgeStrip = { meanLuma: stats.rightMeanLuma, uniformity: stats.rightUniformity }; const letterbox = isMargin(top) && isMargin(bottom); const pillarbox = isMargin(left) && isMargin(right); return letterbox || pillarbox; } /** * True when a Gemini/Google vision key is configured. Used to gate the default * (network) audit — the pipeline integration only runs the audit when a key * exists. Reads the env directly (not the cached key pool) so it stays * deterministic and test-friendly. */ export function hasVisionKey(env: NodeJS.ProcessEnv = process.env): boolean { return [env.GEMINI_API_KEYS, env.GOOGLE_API_KEYS, env.GOOGLE_API_KEY].some( (value) => typeof value === 'string' && value.trim().length > 0, ); } /** * Extract ONE frame from a video to `framePath` (returns it). `fraction` (0–1) * is the position along the clip to sample; it defaults to 0.5 (the midpoint) so * existing single-frame callers/tests are unaffected. */ export type FrameExtractor = ( videoPath: string, framePath: string, fraction?: number, ) => Promise; /** Sample edge-pixel stats from an already-extracted frame image. */ export type EdgeStatsSampler = (framePath: string) => Promise; /** Default number of frames sampled across a clip for mid-clip inspection. */ export const DEFAULT_FRAME_SAMPLE_COUNT = 5; export interface AuditProjectConsistencyOptions { /** Vision client (REQUIRED for tests; defaults to the Gemini-backed client). */ visionClient?: ConsistencyVisionClient; /** * Frame-inspection client for the multi-frame appearing-element / anatomy * check (REQUIRED for tests; defaults to the Gemini two-image client). */ frameInspectionClient?: ConsistencyFrameInspectionClient; /** Frame extractor (defaults to the ffmpeg-backed {@link extractMidFrame}). */ frameExtractor?: FrameExtractor; /** Edge-stats sampler (defaults to the ffmpeg-backed {@link sampleEdgePixelStats}). */ edgeStatsSampler?: EdgeStatsSampler; /** * Number of frames to sample evenly across each clip for mid-clip inspection * (default {@link DEFAULT_FRAME_SAMPLE_COUNT}). Clamped to ≥1. */ frameSampleCount?: number; /** Endpoint override forwarded to the default vision/inspection clients. */ endpoint?: string; /** Explicit Gemini key for the default vision/inspection clients (bypasses the pool). */ keyOverride?: string; /** Injectable fetch for the default vision/inspection clients (offline tests). */ fetcher?: typeof fetch; } const VIDEO_EXTS = ['.mp4', '.mov', '.webm', '.mkv']; const IMAGE_EXTS = ['.png', '.jpg', '.jpeg', '.webp']; /** * Find the representative source media for a scene to audit: * - a rendered output video `outputs/scene-.mp4` (preferred), else * - a rendered output image `outputs/scene-.`, else * - undefined (the scene is skipped — not yet rendered). * * Exported for reuse by the motion-artifact QC (`motion-artifact-qc.ts`), * which audits the same rendered outputs for a different defect class. */ export function resolveSceneMedia( projectDir: string, sceneIndex: number, ): { path: string; kind: 'video' | 'image' } | undefined { const outputsDir = join(projectDir, 'outputs'); for (const ext of VIDEO_EXTS) { const candidate = join(outputsDir, `scene-${sceneIndex}${ext}`); if (existsSync(candidate)) return { path: candidate, kind: 'video' }; } for (const ext of IMAGE_EXTS) { const candidate = join(outputsDir, `scene-${sceneIndex}${ext}`); if (existsSync(candidate)) return { path: candidate, kind: 'image' }; } return undefined; } /** Minimal asset-manifest shape needed to resolve a scene's i2v start image. */ interface AssetManifestAssets { assets?: Array<{ kind?: string; path?: string; sceneIndex?: number }>; } /** * Resolve a scene's i2v START IMAGE (keyframe) from the project's asset-manifest: * the first `image` asset whose `sceneIndex` matches. Mirrors how * `buildExecutionPayload` groups manifest assets by scene. Returns an absolute, * on-disk, existing path or undefined (no manifest, no matching image, a remote * URI, or a missing file — all graceful: the scene simply skips mid-clip * inspection). Pure over the parsed manifest + a fs existence check. */ export function resolveSceneKeyframe( projectDir: string, manifest: AssetManifestAssets, sceneIndex: number, ): string | undefined { for (const asset of manifest.assets ?? []) { if (asset.kind !== 'image') continue; if (asset.sceneIndex !== sceneIndex) continue; const raw = asset.path; if (!raw || typeof raw !== 'string') continue; if (/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) continue; // remote URI — cannot read pixels locally const resolved = isAbsolute(raw) ? raw : resolve(projectDir, raw); if (existsSync(resolved)) return resolved; } return undefined; } /** * The K evenly-spaced sample fractions across a clip for mid-clip inspection. * Interior points only (never 0 or 1): `(i + 1) / (k + 1)` for i in [0, k). So * k=5 → 1/6, 2/6, 3/6, 4/6, 5/6 (≈0.17 … 0.83), spreading samples across the * body of the clip rather than clustering at the ends. Always ≥1 sample. */ export function sampleFractions(k: number): number[] { const count = Math.max(1, Math.floor(k)); const fractions: number[] = []; for (let i = 0; i < count; i += 1) fractions.push((i + 1) / (count + 1)); return fractions; } /** The canonical reference for a profile: first referenceAsset (URL or project-relative). */ function profileReferencePath( workspace: VideoProjectWorkspace, profile: CharacterProfile, ): string | undefined { const ref = profile.referenceAssets?.[0]; if (!ref) return undefined; if (/^[a-z][a-z0-9+.-]*:\/\//i.test(ref)) return ref; // a URI (gobananas://, http(s)://) return isAbsolute(ref) ? ref : resolve(workspace.projectDir, ref); } /** * Audit a project's rendered scenes for character identity/costume consistency. * * Pure aside from the injected/default frame extractor + edge sampler + vision * client. For each storyboard scene with a rendered output, extracts one * mid-frame, runs the deterministic border check, and asks the vision client to * compare each registered scene character against its locked reference + * descriptor + costume. `ok` is false when any checked character mismatches OR * any checked frame has a border. Scenes without a rendered frame are reported * `frameChecked:false` and never flip `ok`. */ export async function auditProjectConsistency( projectSlug: string, root: string = resolveWorkspaceRootFromEnv(), options: AuditProjectConsistencyOptions = {}, ): Promise { const generatedAt = new Date().toISOString(); const workspace = resolveProjectWorkspace(projectSlug, root); const visionClient = options.visionClient ?? createDefaultVisionClient({ ...(options.endpoint ? { endpoint: options.endpoint } : {}), ...(options.keyOverride ? { keyOverride: options.keyOverride } : {}), ...(options.fetcher ? { fetcher: options.fetcher } : {}), }); const frameInspectionClient = options.frameInspectionClient ?? createDefaultFrameInspectionClient({ ...(options.endpoint ? { endpoint: options.endpoint } : {}), ...(options.keyOverride ? { keyOverride: options.keyOverride } : {}), ...(options.fetcher ? { fetcher: options.fetcher } : {}), }); const frameExtractor = options.frameExtractor ?? extractMidFrame; const edgeStatsSampler = options.edgeStatsSampler ?? sampleEdgePixelStats; const frameSampleCount = Math.max(1, Math.floor(options.frameSampleCount ?? DEFAULT_FRAME_SAMPLE_COUNT)); const storyboardPath = artifactPathFor(workspace, 'storyboard'); if (!existsSync(storyboardPath)) { return { projectSlug, ok: true, generatedAt, scenes: [], findings: [] }; } const storyboard = JSON.parse(await readFile(storyboardPath, 'utf-8')) as { scenes?: Array<{ sceneIndex: number; characters?: string[] }>; }; const scenes = [...(storyboard.scenes ?? [])].sort((a, b) => a.sceneIndex - b.sceneIndex); // Asset-manifest holds each scene's i2v start image (the keyframe), used as the // mid-clip-inspection reference. Absent / unparseable → no keyframes (graceful). const assetManifestPath = artifactPathFor(workspace, 'asset-manifest'); let assetManifest: AssetManifestAssets = { assets: [] }; if (existsSync(assetManifestPath)) { try { assetManifest = JSON.parse(await readFile(assetManifestPath, 'utf-8')) as AssetManifestAssets; } catch { assetManifest = { assets: [] }; } } const profiles = await listCharacterProfiles(workspace); const profileByLowerName = new Map(); for (const profile of profiles) { if (profile.name) profileByLowerName.set(profile.name.trim().toLowerCase(), profile); if (profile.id) profileByLowerName.set(profile.id.trim().toLowerCase(), profile); } const sceneResults: ConsistencySceneResult[] = []; const findings: string[] = []; // One temp dir for all extracted frames; cleaned up at the end. const frameDir = await mkdtemp(join(tmpdir(), `vclaw-consistency-${projectSlug}-`)); try { for (const scene of scenes) { const sceneIndex = scene.sceneIndex; const media = resolveSceneMedia(workspace.projectDir, sceneIndex); if (!media) { sceneResults.push({ sceneIndex, frameChecked: false, characters: [], borderDetected: false, extraFigureSuspected: false, appearingElements: [], anatomyIssues: [], framesSampled: 0, }); continue; } // Extract a representative frame (videos -> mid-frame; images -> the image itself). let framePath: string; try { framePath = media.kind === 'video' ? await frameExtractor(media.path, join(frameDir, `scene-${sceneIndex}.png`)) : media.path; } catch { // A frame we cannot extract is advisory, not fatal — record + skip. sceneResults.push({ sceneIndex, frameChecked: false, characters: [], borderDetected: false, extraFigureSuspected: false, appearingElements: [], anatomyIssues: [], framesSampled: 0, }); findings.push(`scene ${sceneIndex}: could not extract a frame from ${media.path} to audit.`); continue; } // Deterministic border/letterbox check. let borderDetected = false; try { borderDetected = detectBorderFromEdgeStats(await edgeStatsSampler(framePath)); } catch { borderDetected = false; // sampling failure is advisory } if (borderDetected) { findings.push(`scene ${sceneIndex}: dark border/margin detected on the rendered frame.`); } // Per-character vision assessment (only registered scene characters). const characters: ConsistencyCharacterResult[] = []; let extraFigureSuspected = false; for (const rawName of scene.characters ?? []) { const name = String(rawName).trim(); if (!name) continue; const profile = profileByLowerName.get(name.toLowerCase()); if (!profile) continue; // unregistered scene characters are out of scope here const refPath = profileReferencePath(workspace, profile); const assessment = await visionClient.assess({ characterName: profile.name, ...(refPath ? { referencePath: refPath } : {}), ...(profile.description ? { description: profile.description } : {}), ...(profile.costume ? { costume: profile.costume } : {}), framePath, sceneIndex, }); characters.push({ name: profile.name, match: assessment.match, issues: assessment.issues }); if (assessment.extraFigure) extraFigureSuspected = true; if (!assessment.match) { const detail = assessment.issues.length > 0 ? `: ${assessment.issues.join('; ')}` : ''; findings.push(`${profile.name} drifts in scene ${sceneIndex}${detail}`); } } if (extraFigureSuspected) { findings.push(`scene ${sceneIndex}: an unexpected extra figure may be present in the frame.`); } // Mid-clip inspection: only for VIDEO clips that have a bound keyframe (the // i2v start image). Sample K frames evenly across the clip and compare each // against the keyframe for elements that appear mid-clip or anatomy errors. const appearingElements: string[] = []; const anatomyIssues: string[] = []; let framesSampled = 0; const keyframePath = media.kind === 'video' ? resolveSceneKeyframe(workspace.projectDir, assetManifest, sceneIndex) : undefined; if (keyframePath) { const fractions = sampleFractions(frameSampleCount); const seenAppearing = new Set(); const seenAnatomy = new Set(); for (let i = 0; i < fractions.length; i += 1) { const fraction = fractions[i] as number; let sampledFramePath: string; try { sampledFramePath = await frameExtractor( media.path, join(frameDir, `scene-${sceneIndex}-f${i}.png`), fraction, ); } catch { continue; // a frame we cannot extract is advisory; try the next } framesSampled += 1; const inspection = await frameInspectionClient.inspect({ keyframePath, framePath: sampledFramePath, sceneIndex, fraction, }); const tag = `t≈${fraction.toFixed(2)}`; for (const raw of inspection.appearingElements ?? []) { const element = String(raw).trim(); if (!element) continue; const tagged = `${tag}: ${element}`; if (seenAppearing.has(tagged)) continue; seenAppearing.add(tagged); appearingElements.push(tagged); } for (const raw of inspection.anatomyIssues ?? []) { const issue = String(raw).trim(); if (!issue) continue; const tagged = `${tag}: ${issue}`; if (seenAnatomy.has(tagged)) continue; seenAnatomy.add(tagged); anatomyIssues.push(tagged); } } for (const element of appearingElements) { findings.push(`scene ${sceneIndex}: element appeared mid-clip (not in keyframe) — ${element}.`); } for (const issue of anatomyIssues) { findings.push(`scene ${sceneIndex}: anatomy/duplication error — ${issue}.`); } } sceneResults.push({ sceneIndex, frameChecked: true, characters, borderDetected, extraFigureSuspected, appearingElements, anatomyIssues, framesSampled, }); } } finally { await rm(frameDir, { recursive: true, force: true }); } const ok = sceneResults.every( (scene) => !scene.borderDetected && scene.characters.every((character) => character.match) && scene.appearingElements.length === 0 && scene.anatomyIssues.length === 0, ); return { projectSlug, ok, generatedAt, scenes: sceneResults, findings }; } // --------------------------------------------------------------------------- // Default (production) frame extractor + edge sampler + vision client. // --------------------------------------------------------------------------- /** * Default ffmpeg-backed frame extractor: probe the video duration, seek to * `fraction` of the way through (default 0.5 = midpoint), and write a single * PNG. The real-spawn path — NOT used in unit tests (they inject a fake * extractor). `fraction` is clamped to [0, 1). */ export async function extractMidFrame( videoPath: string, framePath: string, fraction = 0.5, ): Promise { const clamped = Math.max(0, Math.min(0.999, fraction)); const durationMs = await ffprobeDuration(videoPath).catch(() => 0); const seekSeconds = durationMs > 0 ? (durationMs / 1000) * clamped : 0; await runFfmpeg([ '-ss', seekSeconds.toFixed(3), '-i', videoPath, '-frames:v', '1', framePath, ]); return framePath; } /** * Default ffmpeg-backed edge sampler. Crops the four edge strips and reads each * strip's mean luma + a uniformity proxy (1 - normalized luma std-dev) via * ffmpeg's `signalstats` filter. The real-spawn path — NOT used in unit tests * (they inject a fake sampler). Falls back to a "no border" reading on any * probe failure (advisory, never fatal). */ export async function sampleEdgePixelStats(framePath: string): Promise { // Strip thickness as a fraction of the frame dimension. const strip = '0.06'; const read = async (crop: string): Promise => { const bin = resolveFfmpegBin(); // signalstats exposes YAVG (mean luma) + YDIF/STDEV via metadata; we render // to null and parse the printed metadata. To stay dependency-free and // robust, approximate uniformity from the luma std-dev reported by // `signalstats` (lavfi.signalstats.YDIF is frame-to-frame; for a single // frame we use the spatial proxy via the `entropy`-like YDIF==0, so we // instead derive uniformity from min/max spread). const { spawnSync } = await import('node:child_process'); const probe = spawnSync( bin, [ '-hide_banner', '-i', framePath, '-vf', `${crop},signalstats,metadata=print:file=-`, '-f', 'null', '-', ], { encoding: 'utf-8' }, ); const text = `${probe.stdout ?? ''}\n${probe.stderr ?? ''}`; const num = (key: string): number => { const m = text.match(new RegExp(`lavfi\\.signalstats\\.${key}=([\\d.]+)`)); return m ? Number(m[1]) : NaN; }; const yavg = num('YAVG'); const ymin = num('YMIN'); const ymax = num('YMAX'); const meanLuma = Number.isFinite(yavg) ? yavg : 128; // Uniformity proxy: a flat strip has min≈max. Normalize the spread to [0,1]. const spread = Number.isFinite(ymin) && Number.isFinite(ymax) ? (ymax - ymin) / 255 : 1; const uniformity = Math.max(0, Math.min(1, 1 - spread)); return { meanLuma, uniformity }; }; const [top, bottom, left, right] = await Promise.all([ read(`crop=iw:ih*${strip}:0:0`), read(`crop=iw:ih*${strip}:0:ih*(1-${strip})`), read(`crop=iw*${strip}:ih:0:0`), read(`crop=iw*${strip}:ih:iw*(1-${strip}):0`), ]); return { topMeanLuma: top.meanLuma, bottomMeanLuma: bottom.meanLuma, leftMeanLuma: left.meanLuma, rightMeanLuma: right.meanLuma, topUniformity: top.uniformity, bottomUniformity: bottom.uniformity, leftUniformity: left.uniformity, rightUniformity: right.uniformity, }; } export interface DefaultVisionClientOptions { endpoint?: string; keyOverride?: string; fetcher?: typeof fetch; } /** Build the structured per-character audit prompt for the vision client. */ export function buildConsistencyAuditPrompt(input: ConsistencyVisionInput): string { const lines = [ `You are auditing one frame from scene ${input.sceneIndex} of a video for character consistency.`, `The frame should depict the character "${input.characterName}". A canonical reference image of this character is attached FIRST, the scene frame SECOND.`, '', 'The character must match the reference across BOTH:', ' (a) identity — face, hair, build;', ` (b) costume/wardrobe and its exact colours${input.costume ? ` (locked costume: ${input.costume})` : ''}.`, ]; if (input.description) lines.push(`Reference description: ${input.description}.`); lines.push( '', 'Drift to catch: a garment that changed colour between scenes (e.g. a crimson dhoti rendering tan/pink), facial-hair that appeared or vanished, hairstyle changes, or an unexpected EXTRA person in the frame.', '', 'Reply in this EXACT format (no other text):', 'verdict: ', 'reason: ', 'extra_figure: ', ); return lines.join('\n'); } /** * The default vision client. Reuses the EXISTING shared Gemini-Vision transport * ({@link classifyImageWithGemini} → {@link fetchGeminiWithPool} key pool + * VCLAW_GEMINI_API_ENDPOINT override) used by the assemble QA vision modules — * no new auth path. It classifies the SCENE FRAME (the reference image is * described in the prompt; the shared transport posts a single image) and maps * the `match|mismatch` verdict + reason into a {@link ConsistencyVisionAssessment}. * A transport `error` verdict degrades to a non-fatal "could not assess" issue * (match:true) so a flaky vision call never falsely fails a good render. */ export function createDefaultVisionClient( options: DefaultVisionClientOptions = {}, ): ConsistencyVisionClient { const endpoint = resolveVisionQaEndpoint(options.endpoint); return { async assess(input: ConsistencyVisionInput): Promise { const prompt = buildConsistencyAuditPrompt(input); const classified = await classifyImageWithGemini({ imagePath: input.framePath, prompt, allowedVerdicts: ['match', 'mismatch'] as const, endpoint, ...(options.keyOverride ? { keyOverride: options.keyOverride } : {}), ...(options.fetcher ? { fetcher: options.fetcher } : {}), }); if (classified.verdict === 'error') { // Advisory: an unreadable/failed call must not falsely flag a good render. return { match: true, issues: [] }; } const match = classified.verdict === 'match'; const extraFigure = /extra[_ ]?figure:\s*yes/i.test(classified.reason); return { match, issues: match ? [] : [classified.reason], ...(extraFigure ? { extraFigure: true } : {}), }; }, }; } /** Build the two-image mid-clip inspection prompt (keyframe FIRST, frame SECOND). */ export function buildFrameInspectionPrompt(input: ConsistencyFrameInspectionInput): string { return [ `You are inspecting one sampled frame from scene ${input.sceneIndex} of an image-to-video clip.`, 'The clip was generated from a single START IMAGE (keyframe). Two images are attached:', ' 1. FIRST image = the keyframe (the clip\'s starting reference).', ' 2. SECOND image = a frame sampled later in the clip.', '', 'Compare the SECOND image (the frame) against the FIRST image (the keyframe). Report ONLY clear, obvious defects — when in any doubt, report "none". Two kinds:', ' (A) A NEW SOLID OBJECT that materialized from nothing — e.g. a garland, scarf/veil, weapon, tool, book, or jewellery that a character is suddenly holding/wearing or that floats in mid-air, and that is plainly NOT in the keyframe. Only a distinct, clearly-added solid item counts.', ' (B) ANATOMY/DUPLICATION ERRORS — a person with more than two arms or hands, an extra/duplicate limb, a clearly duplicated copy of a character, or a duplicated object (e.g. two axes where the keyframe has one).', '', 'Do NOT report any of the following — they are EXPECTED and acceptable, never list them: ambient motion or natural movement of anything already in the keyframe; smoke, fire or flame flicker, embers, sparks; birds, insects or animals; wind, dust, mist, fog or motion lines; falling or blowing petals or leaves; flowing, blowing or rippling cloth, sari, dupatta, veil, scarf, robe, hair or water that is already worn or present; glows, halos, light rays, auras, sparkle or colour/lighting shifts; the sun, moon, clouds or sky; shadows; ripples on water; minor pose, hand or facial-expression changes; camera movement or zoom; flags, banners, pennants, parasols, drums, conches or other festival/procession items in a crowd scene; and a rope, noose or lasso already held by a character. A costume merely changing colour is NOT an appearing element.', 'Be conservative: list only what you are highly confident is a genuinely added solid object or a real anatomy/duplication error. If unsure, answer "none".', '', 'Reply in this EXACT format (no other text):', 'appearing: ', 'anatomy: ', ].join('\n'); } /** * Parse the two-line `appearing:` / `anatomy:` reply into a * {@link ConsistencyFrameInspection}. A `none` (case-insensitive) or empty value * yields an empty list; otherwise the value is split on commas/semicolons and * each non-empty item is trimmed. Lenient: missing lines yield empty lists. */ export function parseFrameInspectionReply(text: string): ConsistencyFrameInspection { const pick = (key: string): string[] => { for (const line of text.split('\n')) { const lower = line.toLowerCase().trim(); if (!lower.startsWith(`${key}:`)) continue; const value = line.slice(line.indexOf(':') + 1).trim(); if (!value || /^none\.?$/i.test(value)) return []; return value .split(/[,;]/) .map((part) => part.trim()) .filter((part) => part.length > 0 && !/^none\.?$/i.test(part)); } return []; }; return { appearingElements: pick('appearing'), anatomyIssues: pick('anatomy') }; } /** * The default frame-inspection client. Reuses the SAME Gemini infra as the * identity audit — the two-image {@link classifyTwoImagesWithGemini} call over * the {@link fetchGeminiWithPool} key pool + VCLAW_GEMINI_API_ENDPOINT override — * posting the keyframe + the sampled frame in one call. A transport error * degrades to empty lists (advisory) so a flaky vision call never falsely flags * a clean clip. */ export function createDefaultFrameInspectionClient( options: DefaultVisionClientOptions = {}, ): ConsistencyFrameInspectionClient { const endpoint = resolveVisionQaEndpoint(options.endpoint); return { async inspect(input: ConsistencyFrameInspectionInput): Promise { const prompt = buildFrameInspectionPrompt(input); const result = await classifyTwoImagesWithGemini({ referencePath: input.keyframePath, framePath: input.framePath, prompt, endpoint, ...(options.keyOverride ? { keyOverride: options.keyOverride } : {}), ...(options.fetcher ? { fetcher: options.fetcher } : {}), }); if (result.error) { // Advisory: a failed call must not falsely flag a clean clip. return { appearingElements: [], anatomyIssues: [] }; } return parseFrameInspectionReply(result.text); }, }; }