/** * Pre-render keyframe readiness gate (`vclaw video keyframe-qc`). * * A production series shipped ~30 films whose per-scene keyframes drifted * (wrong crown, mixed image dimensions, missing keyframes) because nothing * machine-checked the keyframes before `produce`. This module is the * mechanical half of that check, productized per working-discipline rule 4: * the TOOL errors instead of relying on an agent remembering. * * Design: * - Pure Node fs — the PNG dimensions come from parsing the IHDR header bytes * directly ({@link parsePngDimensions}); no ImageMagick, no new dependencies, * no provider calls, no spend. * - Given the storyboard's scene count, every scene index i in 0..N-1 must * have `projects//references/scene-keyframe.png`. Missing file, * unreadable/non-PNG file, or a dimension mismatch against the DOMINANT * (most common) WxH is an `error` finding → status 'fail'. A uniform set * whose dominant size is not {@link STANDARD_KEYFRAME_DIMENSIONS} is only an * `advisory` finding → status stays 'pass'. * - The handler (`src/cli/handlers/analysis.ts`) persists the report to * `artifacts/keyframe-qc.json` (schema * `schemas/video/artifacts/keyframe-qc.schema.json`) and exits non-zero on * 'fail' so an `&&` chain or driver halts before a render. */ import { open } from 'node:fs/promises'; import { readFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { join, isAbsolute } from 'node:path'; import { artifactPathFor } from './artifact-store.js'; import { classifyImageWithGemini, resolveVisionQaEndpoint } from './assemble/gemini-vision-classify.js'; import { resolveProjectWorkspace } from './workspace.js'; /** The expected/standard keyframe size; a uniform non-standard set is advisory-only. */ export const STANDARD_KEYFRAME_DIMENSIONS = '1280x720'; /** * Dimensions from a JPEG's first SOF marker. * * PNG-only was not a simplification, it was a blind spot: this gate hardcoded * `references/scene-keyframe.png`, while the 3d-animation-short skill writes * `references/kf-s01.jpg`. The gate therefore could not see a single keyframe of * any film that skill produced — it would have reported all eight scenes missing * — which is why that skill hand-rolled its own inline keyframe check instead of * calling this one. A shared gate nobody can call is not a gate. * * Walks the marker chain rather than scanning for bytes, so an SOF-looking pair * inside EXIF or a thumbnail cannot be mistaken for the real frame header. */ export function parseJpegDimensions(bytes: Uint8Array): { width: number; height: number } | null { if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return null; // not SOI let offset = 2; while (offset + 9 < bytes.length) { if (bytes[offset] !== 0xff) { offset += 1; // fill byte / resync continue; } const marker = bytes[offset + 1]; if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { offset += 2; // standalone markers carry no payload continue; } const length = (bytes[offset + 2] << 8) | bytes[offset + 3]; if (length < 2) return null; // SOF0-3, SOF5-7, SOF9-11, SOF13-15 carry the frame size. DHT (c4), JPG // (c8) and DAC (cc) sit in the same range and must be skipped. const isSof = marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc; if (isSof) { const height = (bytes[offset + 5] << 8) | bytes[offset + 6]; const width = (bytes[offset + 7] << 8) | bytes[offset + 8]; if (width > 0 && height > 0) return { width, height }; return null; } offset += 2 + length; } return null; } /** Dimensions of a PNG or JPEG, whichever the bytes turn out to be. */ export function parseImageDimensions(bytes: Uint8Array): { width: number; height: number } | null { return parsePngDimensions(bytes) ?? parseJpegDimensions(bytes); } /** Machine-readable finding codes emitted by the gate. */ export type KeyframeQcFindingCode = | 'storyboard-missing' | 'keyframe-missing' | 'keyframe-unreadable' | 'keyframe-dimension-mismatch' | 'keyframe-nonstandard-size' | 'keyframe-cast-missing' | 'keyframe-cast-unverified'; export interface KeyframeQcFinding { code: KeyframeQcFindingCode; /** 'error' findings fail the gate; 'advisory' findings never do. */ severity: 'error' | 'advisory'; /** Present for per-scene findings (missing/unreadable/mismatched keyframes). */ sceneIndex?: number; /** Absolute path of the file the finding is about, when file-scoped. */ path?: string; detail: string; } export interface KeyframeQcReport { schemaVersion: 1; projectSlug: string; generatedAt: string; /** Scene count read from artifacts/storyboard.json (0 when the storyboard is missing). */ sceneCount: number; /** Number of scene-keyframe.png files present on disk (readable or not). */ keyframeCount: number; /** The dominant (most common) WxH among readable keyframes, e.g. "1280x720"; null when none. */ dominantDimensions: string | null; findings: KeyframeQcFinding[]; /** 'fail' iff any severity 'error' finding is present. */ status: 'pass' | 'fail'; } const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; /** Signature (8) + IHDR length (4) + "IHDR" (4) + IHDR data (13) + CRC (4). */ const PNG_HEADER_BYTES = 33; /** Upper bound on marker hops when walking a JPEG, so a malformed file cannot loop. */ const JPEG_MAX_SEGMENTS = 64; /** Minimum bytes needed to read width+height (through byte offset 23). */ const PNG_MIN_HEADER_BYTES = 24; /** * Parse width/height from a PNG file's leading bytes (signature + IHDR chunk). * Pure and dependency-free: width is bytes 16-19 big-endian, height 20-23. * Returns null for short buffers, a bad PNG signature, a non-IHDR first chunk, * or zero dimensions (all invalid per the PNG spec). */ export function parsePngDimensions(bytes: Uint8Array): { width: number; height: number } | null { if (bytes.length < PNG_MIN_HEADER_BYTES) return null; for (let i = 0; i < PNG_SIGNATURE.length; i += 1) { if (bytes[i] !== PNG_SIGNATURE[i]) return null; } // First chunk must be IHDR ("IHDR" at bytes 12-15). if (bytes[12] !== 0x49 || bytes[13] !== 0x48 || bytes[14] !== 0x44 || bytes[15] !== 0x52) { return null; } const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); const width = buf.readUInt32BE(16); const height = buf.readUInt32BE(20); if (width === 0 || height === 0) return null; return { width, height }; } /** Expected on-disk keyframe path for a scene: projects//references/scene-keyframe.png. */ export function keyframePathFor(workspaceRoot: string, projectSlug: string, sceneIndex: number): string { const workspace = resolveProjectWorkspace(projectSlug, workspaceRoot); return join(workspace.projectDir, 'references', `scene${sceneIndex}-keyframe.png`); } /** * Where scene `i`'s keyframe actually is, according to the ASSET MANIFEST — * falling back to the legacy `references/scene-keyframe.png` name. * * The manifest is what `buildExecutionPayload` reads when it decides which * image becomes the start frame, so it is the only answer that matches what * will really be submitted. Checking a hardcoded filename instead meant this * gate silently examined nothing on every project that names its keyframes * differently — and then reported a clean pass on the file count it did find. */ export function resolveKeyframePath( projectDir: string, manifest: { assets?: Array<{ kind?: string; sceneIndex?: number; path?: string }> } | null, sceneIndex: number, ): string | null { for (const asset of manifest?.assets ?? []) { if (asset.kind !== 'image' || asset.sceneIndex !== sceneIndex) continue; const raw = asset.path; if (!raw || typeof raw !== 'string') continue; if (/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) continue; // remote URI — no local pixels to read const resolved = isAbsolute(raw) ? raw : join(projectDir, raw); if (existsSync(resolved)) return resolved; } const legacy = join(projectDir, 'references', `scene${sceneIndex}-keyframe.png`); return existsSync(legacy) ? legacy : null; } /** Read the leading bytes of a file without loading the whole image. */ async function readPngHeader(path: string): Promise { try { const handle = await open(path, 'r'); try { const buf = Buffer.alloc(PNG_HEADER_BYTES); const { bytesRead } = await handle.read(buf, 0, PNG_HEADER_BYTES, 0); return buf.subarray(0, bytesRead); } finally { await handle.close(); } } catch { return null; } } /** * JPEG dimensions by WALKING the marker chain with seeks, not by buffering. * * A fixed read window is the wrong shape for JPEG. Each APP segment may be up * to 64KB and there can be several: one real keyframe here carried two ~65KB * APP11 blocks and put its SOF at byte 71,645. A 64KB buffer therefore reported * a perfectly good file as "unreadable" — and any larger constant just moves * the cliff somewhere else. Seeking to each segment header instead costs a * handful of 4-byte reads and is exact regardless of file size. */ async function readJpegDimensionsByWalk(path: string): Promise<{ width: number; height: number } | null> { let handle; try { handle = await open(path, 'r'); } catch { return null; } try { const head = Buffer.alloc(2); const { bytesRead } = await handle.read(head, 0, 2, 0); if (bytesRead < 2 || head[0] !== 0xff || head[1] !== 0xd8) return null; // not SOI let offset = 2; const seg = Buffer.alloc(9); for (let hop = 0; hop < JPEG_MAX_SEGMENTS; hop += 1) { const read = await handle.read(seg, 0, 9, offset); if (read.bytesRead < 4 || seg[0] !== 0xff) return null; const marker = seg[1]; if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { offset += 2; // standalone, no payload continue; } if (marker === 0xda || marker === 0xd9) return null; // scan/end reached without an SOF const length = (seg[2] << 8) | seg[3]; if (length < 2) return null; const isSof = marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc; if (isSof) { if (read.bytesRead < 9) return null; const height = (seg[5] << 8) | seg[6]; const width = (seg[7] << 8) | seg[8]; return width > 0 && height > 0 ? { width, height } : null; } offset += 2 + length; } return null; } finally { await handle.close(); } } /** One character the gate asks the vision model to find in a keyframe. */ export interface KeyframeCastSubject { name: string; /** Visual descriptor from characters.json / the show bible; may be empty. */ description: string; } /** Injectable so the cast check is offline-testable and provider-agnostic. */ export interface KeyframeCastVisionClient { /** True when `subject` is visibly present in the image at `imagePath`. */ isPresent(imagePath: string, subject: KeyframeCastSubject): Promise; } /** * The question put to the vision model, one character at a time. * * Deliberately asks only "is this character VISIBLE", not "does it match the * reference" — identity drift is consistency-audit's job, after a render. This * gate answers the earlier and cheaper question that nothing asked: on an * image-to-video route the keyframe IS the identity lock, so a character listed * on the shot but absent from its start frame gets invented by the model. That * is what produced a shot with tangled extra limbs and the wrong costume, and * no gate looked at it. */ export function buildKeyframeCastPrompt(subject: KeyframeCastSubject): string { const who = subject.description.trim() ? `${subject.name} — ${subject.description.trim()}` : subject.name; return [ 'You are checking a single animation keyframe before it is used as the start frame of a shot.', '', `Is this character VISIBLE anywhere in the image: ${who}`, '', 'Judge presence only. Do not judge quality, style, pose, or how closely it matches any reference.', 'A character partly out of frame or facing away still counts as present.', '', 'Reply on two lines exactly:', 'VERDICT: present|absent', 'REASON: ', ].join('\n'); } /** * Scenes whose keyframe is missing a character the shot lists. * * Pure apart from the injected client, so the whole decision is testable with * no network. A client returning `null` (no key, call failed) yields an * ADVISORY, never a pass and never a false failure — an unverified check that * reports success is the failure mode this file already exists to close. */ export async function findMissingKeyframeCast( scenes: Array<{ sceneIndex: number; keyframePath: string; cast: KeyframeCastSubject[] }>, client: KeyframeCastVisionClient, ): Promise { const findings: KeyframeQcFinding[] = []; for (const scene of scenes) { for (const subject of scene.cast) { const present = await client.isPresent(scene.keyframePath, subject); if (present === null) { findings.push({ code: 'keyframe-cast-unverified', severity: 'advisory', sceneIndex: scene.sceneIndex, path: scene.keyframePath, detail: `scene ${scene.sceneIndex}: could not verify whether "${subject.name}" is in the keyframe (no vision key, or the call failed) — the cast check did NOT run for this character.`, }); continue; } if (!present) { findings.push({ code: 'keyframe-cast-missing', severity: 'error', sceneIndex: scene.sceneIndex, path: scene.keyframePath, detail: `scene ${scene.sceneIndex} lists "${subject.name}" but that character is NOT in its keyframe. On an image-to-video route the keyframe is the identity lock, so the model will invent them — regenerate the keyframe with every listed character visible.`, }); } } } return findings; } /** * Gemini-Vision client for the cast check. Returns null rather than throwing on * an unreadable image or a failed call, so the gate degrades to an advisory * instead of failing a good keyframe. */ export function createKeyframeCastVisionClient( options: { endpoint?: string; keyOverride?: string; fetcher?: typeof fetch } = {}, ): KeyframeCastVisionClient { const endpoint = resolveVisionQaEndpoint(options.endpoint); return { async isPresent(imagePath, subject) { const classified = await classifyImageWithGemini({ imagePath, prompt: buildKeyframeCastPrompt(subject), allowedVerdicts: ['present', 'absent'] as const, endpoint, ...(options.keyOverride ? { keyOverride: options.keyOverride } : {}), ...(options.fetcher ? { fetcher: options.fetcher } : {}), }); if (classified.verdict === 'error') return null; return classified.verdict === 'present'; }, }; } export interface KeyframeQcOptions { /** * Run the vision cast check. Off by default: it costs one call per listed * character per scene, and the rest of this gate is deterministic and free. */ checkCast?: boolean; /** Injectable for tests; defaults to {@link createKeyframeCastVisionClient}. */ castClient?: KeyframeCastVisionClient; env?: NodeJS.ProcessEnv; } /** Cast for each scene, joined to its keyframe and to any known descriptor. */ function buildCastSubjects( scenes: unknown[], descriptions: Map, ): Map { const out = new Map(); scenes.forEach((raw, index) => { const scene = raw as { sceneIndex?: number; characters?: unknown } | null; const names = Array.isArray(scene?.characters) ? scene.characters : []; const cast: KeyframeCastSubject[] = []; for (const value of names) { const name = typeof value === 'string' ? value.trim() : ''; if (!name) continue; cast.push({ name, description: descriptions.get(name.toLowerCase()) ?? '' }); } if (cast.length > 0) out.set(scene?.sceneIndex ?? index, cast); }); return out; } /** * Run the fail-fast pre-render keyframe readiness gate for a project. * Deterministic, read-only, no provider calls. See the module doc for the * finding taxonomy; status is 'fail' iff any 'error' finding is present. */ export async function runKeyframeQc( workspaceRoot: string, projectSlug: string, options: KeyframeQcOptions = {}, ): Promise { const workspace = resolveProjectWorkspace(projectSlug, workspaceRoot); const generatedAt = new Date().toISOString(); const findings: KeyframeQcFinding[] = []; const storyboardPath = artifactPathFor(workspace, 'storyboard'); let scenes: unknown[] | null = null; try { const parsed = JSON.parse(await readFile(storyboardPath, 'utf-8')) as { scenes?: unknown } | null; if (parsed && Array.isArray(parsed.scenes)) scenes = parsed.scenes; } catch { scenes = null; } if (scenes === null) { findings.push({ code: 'storyboard-missing', severity: 'error', path: storyboardPath, detail: 'artifacts/storyboard.json is missing or unparseable (no scenes array) — run `vclaw video storyboard` first.', }); return { schemaVersion: 1, projectSlug, generatedAt, sceneCount: 0, keyframeCount: 0, dominantDimensions: null, findings, status: 'fail', }; } const sceneCount = scenes.length; const referencesDir = join(workspace.projectDir, 'references'); const readable: Array<{ sceneIndex: number; path: string; dimensions: string }> = []; let keyframeCount = 0; // The manifest is the source of truth for which image becomes scene i's start // frame; the legacy filename is only a fallback. See resolveKeyframePath. let manifest: { assets?: Array<{ kind?: string; sceneIndex?: number; path?: string }> } | null = null; try { manifest = JSON.parse(await readFile(artifactPathFor(workspace, 'asset-manifest'), 'utf-8')); } catch { manifest = null; } for (let i = 0; i < sceneCount; i += 1) { const path = resolveKeyframePath(workspace.projectDir, manifest, i); if (!path) { findings.push({ code: 'keyframe-missing', severity: 'error', sceneIndex: i, path: join(referencesDir, `scene${i}-keyframe.png`), detail: `scene ${i} has no keyframe: no image asset for that scene in the asset manifest, and no references/scene${i}-keyframe.png.`, }); continue; } keyframeCount += 1; const header = await readPngHeader(path); const dims = (header ? parsePngDimensions(header) : null) ?? (await readJpegDimensionsByWalk(path)); if (!dims) { findings.push({ code: 'keyframe-unreadable', severity: 'error', sceneIndex: i, path, detail: `scene ${i} keyframe is not a readable PNG or JPEG (bad signature, or a truncated IHDR/SOF header).`, }); continue; } readable.push({ sceneIndex: i, path, dimensions: `${dims.width}x${dims.height}` }); } // Dominant (most common) WxH; ties break to the first-seen dimensions in // scene order so the result is deterministic. // Cast check (opt-in). Runs on the SAME resolved keyframes the dimension // checks used, so it can never disagree about which file is scene i's start // frame — the disagreement that let this whole class of bug through. if (options.checkCast) { const descriptions = new Map(); try { const raw = JSON.parse( await readFile(join(workspace.projectDir, 'characters', 'characters.json'), 'utf-8'), ) as { characters?: Array<{ name?: string; description?: string; costume?: string }> }; for (const c of raw.characters ?? []) { if (!c?.name) continue; descriptions.set( c.name.toLowerCase(), [c.description, c.costume].filter((x) => typeof x === 'string' && x.trim()).join('. '), ); } } catch { // No profiles is fine — the model still gets the character's name. } const castByScene = buildCastSubjects(scenes, descriptions); const targets = readable .filter((entry) => (castByScene.get(entry.sceneIndex)?.length ?? 0) > 0) .map((entry) => ({ sceneIndex: entry.sceneIndex, keyframePath: entry.path, cast: castByScene.get(entry.sceneIndex) ?? [], })); if (targets.length > 0) { const client = options.castClient ?? createKeyframeCastVisionClient(); findings.push(...(await findMissingKeyframeCast(targets, client))); } } const counts = new Map(); for (const entry of readable) counts.set(entry.dimensions, (counts.get(entry.dimensions) ?? 0) + 1); let dominantDimensions: string | null = null; let bestCount = 0; for (const [dimensions, count] of counts) { if (count > bestCount) { dominantDimensions = dimensions; bestCount = count; } } if (dominantDimensions !== null) { for (const entry of readable) { if (entry.dimensions !== dominantDimensions) { findings.push({ code: 'keyframe-dimension-mismatch', severity: 'error', sceneIndex: entry.sceneIndex, path: entry.path, detail: `scene ${entry.sceneIndex} keyframe is ${entry.dimensions} but the dominant keyframe size is ${dominantDimensions}.`, }); } } if (dominantDimensions !== STANDARD_KEYFRAME_DIMENSIONS) { findings.push({ code: 'keyframe-nonstandard-size', severity: 'advisory', detail: `dominant keyframe size ${dominantDimensions} is not the standard ${STANDARD_KEYFRAME_DIMENSIONS} — fine if intentional (e.g. 9:16), but verify against the project aspect ratio.`, }); } } const status: KeyframeQcReport['status'] = findings.some((f) => f.severity === 'error') ? 'fail' : 'pass'; return { schemaVersion: 1, projectSlug, generatedAt, sceneCount, keyframeCount, dominantDimensions, findings, status, }; }