/** * Motion-artifact vision QC — the dense-frame audit for RENDERING artifacts. * * Operator feedback baked into the tool (The Counsel brand film, 2026-06-27): * a still-frame + whisper QC pass is BLIND to motion artifacts — the defects a * human catches at full playback. The four production-observed classes: * 1. BREATH VAPOUR — a condensation puff near a speaker's mouth/nose, as if * exhaling cold air in a scene that is not freezing. * 2. GRAIN-AS-FOG — film grain / sensor noise reading as a drifting fog or * smoke layer over dark/shadow regions. * 3. MORPHING — an object, face or limb smearing/melting into a different * shape between moments of the clip. * 4. VANISHING PROPS — a solid object present at one moment that is simply * absent moments later without leaving frame. * * This is deliberately DISTINCT from `consistency-audit` (identity/costume * drift + keyframe-anchored appearing-element checks): that audit's inspection * prompt EXCLUDES mist/fog/smoke/vapour to avoid false positives on ambient * motion — which is exactly why classes 1–2 slip through it — and it never * compares ADJACENT samples, which is what classes 3–4 need. This module fills * that gap with two inspection lanes over K densely-sampled frames per clip: * - a PER-FRAME lane (single image → statically-visible vapour / grain-fog), and * - an ADJACENT-PAIR lane (frame i vs frame i+1 → morphing / vanished / * TEMPORAL vapour wisps; the scene keyframe, when bound, acts as the pair * anchor before the first sample so a defect right at the start is still * caught). The temporal vapour class exists because the ground-truth * scene08-breath clip proved some vapour is invisible in every single * frame and only reads as a difference BETWEEN frames. * * Design (mirrors consistency-audit exactly): * - The CORE ({@link auditMotionArtifacts}) is pure + injectable: frame * extractor, per-frame client, and pair client are all injectable; tests * never touch ffmpeg or the network. Aggregation (t≈-tagging + dedup) is * deterministic. * - The DEFAULT clients reuse the EXISTING shared Gemini-Vision transport * ({@link classifyImageWithGemini} / {@link classifyTwoImagesWithGemini} over * the env-backed key pool + VCLAW_GEMINI_API_ENDPOINT override). No new auth * path. Transport failures degrade to empty findings (advisory) so a flaky * vision call never falsely fails a good render. * - Prompts are CONSERVATIVE (report only high-confidence defects, "none" when * unsure) because this gate runs before a render is presented as done. */ import { existsSync } from 'node:fs'; import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { artifactPathFor } from './artifact-store.js'; import { resolveProjectWorkspace } from './workspace.js'; import { resolveWorkspaceRootFromEnv } from './workspace-root.js'; import { classifyImageWithGemini, classifyTwoImagesWithGemini, resolveVisionQaEndpoint, } from './assemble/gemini-vision-classify.js'; import { extractMidFrame, resolveSceneKeyframe, resolveSceneMedia, sampleFractions, type FrameExtractor, } from './consistency-audit.js'; /** Per-frame artifacts from the single-image lane. */ export interface MotionFrameArtifacts { /** Breath-vapour / condensation puffs near a mouth or nose. Empty when clean. */ breathVapour: string[]; /** Film grain / noise reading as drifting fog over dark regions. Empty when clean. */ grainFog: string[]; } /** Input passed to the per-frame client for one sampled frame. */ export interface MotionFrameInput { /** Path to the extracted sampled frame. */ framePath: string; /** Scene index (for context / prompts). */ sceneIndex: number; /** Fraction (0–1) of the clip at which this frame was sampled. */ fraction: number; } /** Injectable per-frame client — the default reuses the Gemini infra; tests inject a fake. */ export interface MotionFrameClient { inspectFrame(input: MotionFrameInput): Promise; } /** Pair artifacts from the adjacent-frame lane. */ export interface MotionPairArtifacts { /** Objects/faces/limbs that smeared or transformed between the two frames. Empty when clean. */ morphing: string[]; /** Solid objects present in the earlier frame but absent in the later one. Empty when clean. */ vanishedElements: string[]; /** * Vapour/mist wisps near a face that appear, move, or change shape BETWEEN * the two frames — the temporal-contrast catch for breath vapour that is too * faint/ambiguous to call in any single frame (the ground-truth * scene08-breath miss: 13 hand-reviewed stills showed nothing conclusive, * yet the vapour is obvious at playback). Merged into the scene's * breathVapour findings with a pair-window tag. Empty when clean. */ vapourWisps: string[]; } /** Input passed to the pair client for one adjacent frame pair. */ export interface MotionPairInput { /** Path to the EARLIER image (a sampled frame, or the scene keyframe for the first pair). */ earlierFramePath: string; /** Path to the LATER sampled frame. */ laterFramePath: string; /** Scene index (for context / prompts). */ sceneIndex: number; /** Fraction of the earlier image (0 when it is the keyframe). */ earlierFraction: number; /** Fraction of the later frame. */ laterFraction: number; /** * The scene's storyboard description, when available. Grounds the morphing * check in SCENE INTENT: a transformation the description explicitly calls * for (an object materializing / forging / assembling / folding / sealing) * is the scene working as directed, not a rendering defect. Production * evidence (hermes-do-launch, 2026-07-04): a helmet forged from circuit * traces and a shield folding shut were both false-flagged as morphing by * the intent-blind prompt. Absent → the prompt is byte-identical to legacy. */ sceneDescription?: string; } /** Injectable pair client — the default reuses the Gemini two-image transport. */ export interface MotionPairClient { inspectPair(input: MotionPairInput): Promise; } /** Per-scene motion-QC result. */ export interface MotionQcSceneResult { sceneIndex: number; /** True when the scene had a rendered VIDEO clip that was actually sampled. */ clipChecked: boolean; /** Number of frames actually extracted + inspected. */ framesSampled: number; /** Breath-vapour findings, deduped, tagged with the frame fraction (e.g. "t≈0.40: …"). */ breathVapour: string[]; /** Grain-as-fog findings, deduped, tagged with the frame fraction. */ grainFog: string[]; /** Morphing findings, deduped, tagged with the pair window (e.g. "t≈0.20→0.40: …"). */ morphing: string[]; /** Vanished-object findings, deduped, tagged with the pair window. */ vanishedElements: string[]; } /** The full structured report (also persisted to artifacts/motion-artifact-qc.json). */ export interface MotionQcReport { projectSlug: string; /** False when any scene has any motion-artifact finding. */ ok: boolean; generatedAt: string; scenes: MotionQcSceneResult[]; /** Flat, operator-facing list of every artifact found across all scenes. */ findings: string[]; } /** * Default number of frames sampled per clip. Deliberately DENSER than the * consistency-audit default (5): motion artifacts live between moments, so the * pair lane needs tighter windows to catch a morph or a vanish. */ export const DEFAULT_MOTION_QC_SAMPLE_COUNT = 9; export interface AuditMotionArtifactsOptions { /** Per-frame client (REQUIRED for tests; defaults to the Gemini-backed client). */ frameClient?: MotionFrameClient; /** Pair client (REQUIRED for tests; defaults to the Gemini two-image client). */ pairClient?: MotionPairClient; /** Frame extractor (defaults to the ffmpeg-backed extractMidFrame). */ frameExtractor?: FrameExtractor; /** Frames sampled per clip (default {@link DEFAULT_MOTION_QC_SAMPLE_COUNT}). Clamped to ≥2. */ sampleCount?: number; /** Endpoint override forwarded to the default clients. */ endpoint?: string; /** Explicit Gemini key for the default clients (bypasses the pool). */ keyOverride?: string; /** Injectable fetch for the default clients (offline tests). */ fetcher?: typeof fetch; } /** Minimal asset-manifest shape needed to resolve a scene's keyframe. */ interface AssetManifestAssets { assets?: Array<{ kind?: string; path?: string; sceneIndex?: number }>; } const tagFraction = (fraction: number): string => `t≈${fraction.toFixed(2)}`; /** * True when a parsed finding fragment is the vision model ECHOING the reply * template rather than reporting a defect (production evidence, * hermes-do-launch 2026-07-04: `vapour: ` * echoed verbatim parsed into two junk findings: ``). Real findings describe visible content and never contain * angle-bracket placeholders; fragments of the literal template text are * likewise noise. */ function isTemplateEcho(part: string): boolean { if (part.includes('<') || part.includes('>')) return true; const lower = part.toLowerCase(); return lower.includes('comma-separated') || /^or\s+"?none"?\.?$/.test(lower); } /** Dedup + tag helper shared by both lanes. */ function collectTagged(target: string[], seen: Set, tag: string, items: string[]): void { for (const raw of items ?? []) { const item = String(raw).trim(); if (!item) continue; const tagged = `${tag}: ${item}`; if (seen.has(tagged)) continue; seen.add(tagged); target.push(tagged); } } /** * Audit a project's rendered clips for motion artifacts (breath vapour, * grain-as-fog, morphing, vanishing props). * * Pure aside from the injected/default frame extractor + vision clients. For * each storyboard scene with a rendered VIDEO output, extracts K dense frames, * runs the per-frame lane on each and the pair lane on each adjacent pair * (anchored on the scene keyframe when one is bound). Scenes without a rendered * clip (or with an image-only output) are reported `clipChecked:false` and * never flip `ok`. */ export async function auditMotionArtifacts( projectSlug: string, root: string = resolveWorkspaceRootFromEnv(), options: AuditMotionArtifactsOptions = {}, ): Promise { const generatedAt = new Date().toISOString(); const workspace = resolveProjectWorkspace(projectSlug, root); const frameClient = options.frameClient ?? createDefaultMotionFrameClient({ ...(options.endpoint ? { endpoint: options.endpoint } : {}), ...(options.keyOverride ? { keyOverride: options.keyOverride } : {}), ...(options.fetcher ? { fetcher: options.fetcher } : {}), }); const pairClient = options.pairClient ?? createDefaultMotionPairClient({ ...(options.endpoint ? { endpoint: options.endpoint } : {}), ...(options.keyOverride ? { keyOverride: options.keyOverride } : {}), ...(options.fetcher ? { fetcher: options.fetcher } : {}), }); const frameExtractor = options.frameExtractor ?? extractMidFrame; const sampleCount = Math.max(2, Math.floor(options.sampleCount ?? DEFAULT_MOTION_QC_SAMPLE_COUNT)); const storyboardPath = artifactPathFor(workspace, 'storyboard'); if (!existsSync(storyboardPath)) { return { projectSlug, ok: true, generatedAt, scenes: [], findings: [] }; } const storyboard = JSON.parse(await readFile(storyboardPath, 'utf-8')) as { scenes?: Array<{ sceneIndex: number; description?: string }>; }; const scenes = [...(storyboard.scenes ?? [])].sort((a, b) => a.sceneIndex - b.sceneIndex); // Asset-manifest holds each scene's i2v start image (the keyframe), used as // the pair-lane anchor. Absent / unparseable → no anchor (graceful). const assetManifestPath = artifactPathFor(workspace, 'asset-manifest'); let assetManifest: AssetManifestAssets = { assets: [] }; if (existsSync(assetManifestPath)) { try { assetManifest = JSON.parse(await readFile(assetManifestPath, 'utf-8')) as AssetManifestAssets; } catch { assetManifest = { assets: [] }; } } const sceneResults: MotionQcSceneResult[] = []; const findings: string[] = []; const frameDir = await mkdtemp(join(tmpdir(), `vclaw-motion-qc-${projectSlug}-`)); try { for (const scene of scenes) { const sceneIndex = scene.sceneIndex; const media = resolveSceneMedia(workspace.projectDir, sceneIndex); if (!media || media.kind !== 'video') { // Motion artifacts only exist in motion — images and unrendered scenes skip. sceneResults.push({ sceneIndex, clipChecked: false, framesSampled: 0, breathVapour: [], grainFog: [], morphing: [], vanishedElements: [], }); continue; } // Extract the dense sample set (extraction failures are advisory per frame). const fractions = sampleFractions(sampleCount); const sampled: Array<{ fraction: number; path: string }> = []; for (let i = 0; i < fractions.length; i += 1) { const fraction = fractions[i] as number; try { const path = await frameExtractor( media.path, join(frameDir, `scene-${sceneIndex}-f${i}.png`), fraction, ); sampled.push({ fraction, path }); } catch { continue; } } if (sampled.length === 0) { sceneResults.push({ sceneIndex, clipChecked: false, framesSampled: 0, breathVapour: [], grainFog: [], morphing: [], vanishedElements: [], }); findings.push(`scene ${sceneIndex}: could not extract any frame from ${media.path} to QC.`); continue; } const breathVapour: string[] = []; const grainFog: string[] = []; const morphing: string[] = []; const vanishedElements: string[] = []; const seenVapour = new Set(); const seenFog = new Set(); const seenMorph = new Set(); const seenVanished = new Set(); // Lane 1: per-frame (vapour / grain-fog). for (const frame of sampled) { const verdict = await frameClient.inspectFrame({ framePath: frame.path, sceneIndex, fraction: frame.fraction, }); const tag = tagFraction(frame.fraction); collectTagged(breathVapour, seenVapour, tag, verdict.breathVapour); collectTagged(grainFog, seenFog, tag, verdict.grainFog); } // Lane 2: adjacent pairs (morphing / vanished), anchored on the keyframe // when one is bound so a defect right at the clip start is still caught. const keyframePath = resolveSceneKeyframe(workspace.projectDir, assetManifest, sceneIndex); const pairChain: Array<{ fraction: number; path: string }> = keyframePath ? [{ fraction: 0, path: keyframePath }, ...sampled] : sampled; for (let i = 0; i + 1 < pairChain.length; i += 1) { const earlier = pairChain[i] as { fraction: number; path: string }; const later = pairChain[i + 1] as { fraction: number; path: string }; const verdict = await pairClient.inspectPair({ earlierFramePath: earlier.path, laterFramePath: later.path, sceneIndex, earlierFraction: earlier.fraction, laterFraction: later.fraction, ...(scene.description ? { sceneDescription: scene.description } : {}), }); const tag = `t≈${earlier.fraction.toFixed(2)}→${later.fraction.toFixed(2)}`; collectTagged(morphing, seenMorph, tag, verdict.morphing); collectTagged(vanishedElements, seenVanished, tag, verdict.vanishedElements); // Temporal vapour merges into the same breathVapour bucket as the // per-frame lane — one operator-facing category, two detectors. collectTagged(breathVapour, seenVapour, tag, verdict.vapourWisps); } for (const item of breathVapour) { findings.push(`scene ${sceneIndex}: breath-vapour artifact — ${item}.`); } for (const item of grainFog) { findings.push(`scene ${sceneIndex}: grain-as-fog artifact — ${item}.`); } for (const item of morphing) { findings.push(`scene ${sceneIndex}: morphing artifact — ${item}.`); } for (const item of vanishedElements) { findings.push(`scene ${sceneIndex}: vanished mid-clip — ${item}.`); } sceneResults.push({ sceneIndex, clipChecked: true, framesSampled: sampled.length, breathVapour, grainFog, morphing, vanishedElements, }); } } finally { await rm(frameDir, { recursive: true, force: true }); } const ok = sceneResults.every( (scene) => scene.breathVapour.length === 0 && scene.grainFog.length === 0 && scene.morphing.length === 0 && scene.vanishedElements.length === 0, ); return { projectSlug, ok, generatedAt, scenes: sceneResults, findings }; } // --------------------------------------------------------------------------- // Default (production) Gemini-backed clients. // --------------------------------------------------------------------------- export interface DefaultMotionClientOptions { endpoint?: string; keyOverride?: string; fetcher?: typeof fetch; } /** Build the single-frame vapour / grain-fog prompt. */ export function buildMotionFramePrompt(input: MotionFrameInput): string { return [ `You are QC-inspecting one frame sampled at ${Math.round(input.fraction * 100)}% of an AI-generated video clip (scene ${input.sceneIndex}).`, 'Look ONLY for these two RENDERING artifacts. Report each only when you are highly confident; when in any doubt, answer "none".', ' (A) BREATH VAPOUR — a visible puff or cloud of condensation at a person\'s mouth or nose, as if exhaling cold air. Report it unless the scene is unmistakably freezing outdoors (visible snow/ice/frost).', ' (B) GRAIN-AS-FOG — film grain or sensor noise rendered as a hazy, drifting FOG or smoke veil over dark or shadow regions: a speckled/noisy grey layer sitting on top of blacks. Deliberate volumetric scene fog (a smooth atmospheric haze with depth, light shafts, or a weather context) is NOT a defect — only noise-textured haze over dark areas counts.', '', 'Reply in this EXACT format (no other text):', 'verdict: ', 'reason: vapour= | grain_fog=', ].join('\n'); } /** * Parse the single-line `vapour=… | grain_fog=…` reason payload into a * {@link MotionFrameArtifacts}. Lenient: a missing key or a `none` value yields * an empty list. */ export function parseMotionFrameReason(reason: string): MotionFrameArtifacts { const pick = (key: string): string[] => { const match = reason.match(new RegExp(`${key}\\s*=\\s*([^|]*)`, 'i')); if (!match) return []; const value = (match[1] ?? '').trim(); if (!value || /^none\.?$/i.test(value)) return []; return value .split(/[,;]/) .map((part) => part.trim()) .filter((part) => part.length > 0 && !/^none\.?$/i.test(part) && !isTemplateEcho(part)); }; return { breathVapour: pick('vapour'), grainFog: pick('grain_fog') }; } /** * The default per-frame client over the shared single-image Gemini transport. * A `clean` verdict short-circuits to empty lists; a transport `error` degrades * to empty lists (advisory) so a flaky call never falsely flags a clean clip. */ export function createDefaultMotionFrameClient( options: DefaultMotionClientOptions = {}, ): MotionFrameClient { const endpoint = resolveVisionQaEndpoint(options.endpoint); return { async inspectFrame(input: MotionFrameInput): Promise { const classified = await classifyImageWithGemini({ imagePath: input.framePath, prompt: buildMotionFramePrompt(input), allowedVerdicts: ['clean', 'artifacts'] as const, endpoint, ...(options.keyOverride ? { keyOverride: options.keyOverride } : {}), ...(options.fetcher ? { fetcher: options.fetcher } : {}), }); if (classified.verdict !== 'artifacts') { return { breathVapour: [], grainFog: [] }; // clean or advisory error } return parseMotionFrameReason(classified.reason); }, }; } /** Build the two-image adjacent-pair morphing / vanished prompt. */ export function buildMotionPairPrompt(input: MotionPairInput): string { const earlierLabel = input.earlierFraction === 0 ? 'the clip\'s START IMAGE (keyframe)' : `the frame sampled at ${Math.round(input.earlierFraction * 100)}% of the clip`; // SCENE INTENT grounding (additive; absent description → byte-identical // legacy prompt): transformation scenes — an object forging, materializing, // assembling, folding shut — are the scene WORKING, and an intent-blind // morphing check false-flags exactly those (hermes-do-launch 2026-07-04: // helmet-forge + shield-seal scenes both flagged). const intentBlock = input.sceneDescription?.trim() ? [ '', `SCENE INTENT (what this scene is DIRECTED to show): "${input.sceneDescription.trim()}"`, 'A transformation that the SCENE INTENT explicitly calls for — an object materializing, forging, assembling, dissolving, folding, or sealing as described — is INTENDED and must NOT be reported as morphing. Report morphing only for changes the scene intent does NOT call for.', ] : []; return [ `You are QC-inspecting two moments of the SAME AI-generated video clip (scene ${input.sceneIndex}).`, `Two images are attached: FIRST = ${earlierLabel}; SECOND = the frame sampled at ${Math.round(input.laterFraction * 100)}% of the clip.`, ...intentBlock, '', 'Compare the SECOND image against the FIRST. Look ONLY for these three RENDERING artifacts. For (A) and (B), report only when you are highly confident; when in any doubt, answer "none".', ' (A) MORPHING — an object, prop, face, or limb that has smeared, melted, or transformed into a DIFFERENT thing or shape between the two moments (e.g. a sword becoming a staff, a hand melting into cloth, a face warping into different anatomy). Normal pose changes, expression changes, and camera movement are NOT morphing.', ' (B) VANISHED OBJECT — a distinct solid object clearly present in the FIRST image that is simply ABSENT in the SECOND without having exited the frame (e.g. a held cup that is gone, a necklace that disappeared). An object that is merely occluded by a body/hand, moved out of view by camera motion, or plausibly put down is NOT vanished.', ' (C) VAPOUR WISP — a faint plume, wisp, or haze of vapour/mist/smoke near a person\'s mouth or nose that APPEARS, MOVES, or CHANGES SHAPE between the two images. Judge this one by DIFFERENCE: even if the haze is too faint to call in either image alone, a wispy patch near a face whose shape or position clearly differs between the two frames counts. Do NOT report: steam from a visible hot drink/food, a cigarette/vape a character is deliberately using, or static background haze that is identical in both frames.', '', 'Reply in this EXACT format (no other text):', 'morphing: ', 'vanished: ', 'vapour: ', ].join('\n'); } /** * Parse the three-line `morphing:` / `vanished:` / `vapour:` reply into a * {@link MotionPairArtifacts}. Lenient: missing lines / `none` yield empty lists * (so a client on the old two-line contract still parses cleanly). */ export function parseMotionPairReply(text: string): MotionPairArtifacts { const pick = (key: string): string[] => { for (const line of text.split('\n')) { const lower = line.toLowerCase().trim(); if (!lower.startsWith(`${key}:`)) continue; const value = line.slice(line.indexOf(':') + 1).trim(); if (!value || /^none\.?$/i.test(value)) return []; return value .split(/[,;]/) .map((part) => part.trim()) .filter((part) => part.length > 0 && !/^none\.?$/i.test(part) && !isTemplateEcho(part)); } return []; }; return { morphing: pick('morphing'), vanishedElements: pick('vanished'), vapourWisps: pick('vapour') }; } /** * The default pair client over the shared two-image Gemini transport. A * transport error degrades to empty lists (advisory). */ export function createDefaultMotionPairClient( options: DefaultMotionClientOptions = {}, ): MotionPairClient { const endpoint = resolveVisionQaEndpoint(options.endpoint); return { async inspectPair(input: MotionPairInput): Promise { const result = await classifyTwoImagesWithGemini({ referencePath: input.earlierFramePath, framePath: input.laterFramePath, prompt: buildMotionPairPrompt(input), endpoint, ...(options.keyOverride ? { keyOverride: options.keyOverride } : {}), ...(options.fetcher ? { fetcher: options.fetcher } : {}), }); if (result.error) { return { morphing: [], vanishedElements: [], vapourWisps: [] }; } return parseMotionPairReply(result.text); }, }; }