import { safeErrorBody } from './http-error-safety.js'; import { readFile, mkdtemp, readdir, rm } from 'node:fs/promises'; import { existsSync, statSync } from 'node:fs'; import { execFile } from 'node:child_process'; import { extname, join } from 'node:path'; import { tmpdir } from 'node:os'; import { promisify } from 'node:util'; import { createAnalyzeOutput } from './analyze-output.js'; import { probeMedia } from './final-media.js'; import { fetchGeminiWithPool } from './gemini-key-pool.js'; import type { VideoAnalyzeOutput } from './types.js'; const execFileAsync = promisify(execFile); const DEFAULT_GEMINI_ANALYZE_ENDPOINT = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent'; /** Number of JPEG frames sampled (evenly across the clip) from a local source. */ const ANALYZE_FRAME_COUNT = 6; /** * Pure sampling-math seam: given a clip duration in seconds, produce the ffmpeg * `-vf` value and frame cap that spread {@link count} frames EVENLY across the * whole clip (one frame every `duration / count` seconds via the `fps` filter), * not clustered at the head. Returns `null` when the duration is unusable * (missing/non-finite/≤0) so the caller can fall back to head-clustered * `thumbnail` sampling without breaking the local-file path. * * The `fps` filter can emit `count` or `count ± 1` frames at clip boundaries, so * the caller MUST also pass `-frames:v frames` and tolerate fewer than `count` * outputs (the caller already slices/handles the resulting array). */ export function analyzeFrameSampling( durationSeconds: number | undefined, count = ANALYZE_FRAME_COUNT, ): { vf: string; frames: number } | null { if ( typeof durationSeconds !== 'number' || !Number.isFinite(durationSeconds) || durationSeconds <= 0 || count <= 0 ) { return null; } // A frame every duration/count seconds → `count` frames spread across [0, D). return { vf: `fps=${count}/${durationSeconds},scale=512:-1`, frames: count }; } /** * Extracts up to {@link ANALYZE_FRAME_COUNT} downscaled JPEG frames spread * EVENLY across a local video file via ffmpeg and returns them base64-encoded. * * When the clip duration is known (passed in or probed via {@link probeMedia}), * sampling uses the `fps=N/D` filter so frames cover the whole clip rather than * clustering in the first ~20s the way `thumbnail` + `-frames:v` does. If the * duration cannot be determined, it falls back to the legacy `thumbnail` * sampling so the local-file path never breaks. The temp directory is always * removed in the `finally`. Callers must guarantee the path is a readable local * file; failures (ffmpeg missing, corrupt input) propagate so the live path can * fall back to text-only analysis. */ async function extractAnalyzeFrames( videoPath: string, durationSeconds?: number, ): Promise { const ffmpegBin = process.env.VCLAW_FFMPEG_BIN ?? 'ffmpeg'; // Resolve the clip duration: prefer the caller-supplied value, otherwise probe // it. A failed probe is non-fatal — we just lose even spreading and fall back // to head-clustered `thumbnail` sampling. let resolvedDuration = durationSeconds; if (analyzeFrameSampling(resolvedDuration) === null) { try { const probe = await probeMedia(videoPath); resolvedDuration = probe.durationSeconds; } catch { resolvedDuration = undefined; } } const sampling = analyzeFrameSampling(resolvedDuration); const dir = await mkdtemp(join(tmpdir(), 'vclaw-analyze-frames-')); try { // Even path: `fps=N/D` emits one frame every D/N seconds (spread across the // whole clip); `-frames:v N` caps the count. Fallback path: `thumbnail` // picks one representative frame per window (head-clustered) — only used when // the duration is unknown so even sampling is impossible. const vf = sampling ? sampling.vf : 'thumbnail,scale=512:-1'; const frameCap = sampling ? sampling.frames : ANALYZE_FRAME_COUNT; await execFileAsync( ffmpegBin, [ '-v', 'error', '-i', videoPath, '-vf', vf, '-frames:v', String(frameCap), '-vsync', 'vfr', join(dir, 'frame-%02d.jpg'), ], { encoding: 'buffer' }, ); const entries = (await readdir(dir)) .filter((name) => name.toLowerCase().endsWith('.jpg')) .sort(); if (entries.length === 0) { throw new Error('ffmpeg produced no frames for analyze ingestion.'); } const frames: string[] = []; // `fps` can yield N±1 at boundaries; cap at ANALYZE_FRAME_COUNT either way. for (const name of entries.slice(0, ANALYZE_FRAME_COUNT)) { const bytes = await readFile(join(dir, name)); frames.push(bytes.toString('base64')); } return frames; } finally { await rm(dir, { recursive: true, force: true }); } } const ANALYZE_PROMPT = `You analyze reference videos for reusable ad and short-form video templates. Return ONLY valid JSON with this exact shape: { "pacing": { "label": "slow|medium|fast|mixed", "notes": ["..."] }, "structure": { "hook": "...", "beats": ["...", "..."], "ending": "..." }, "motionClassification": { "primaryMode": "motion-clips|animated-stills|mixed|unknown", "notes": ["..."] }, "keep": ["..."], "change": ["..."], "reusableVariables": ["..."], "styleLayers": ["..."], "beatCompression": { "targetDurationSeconds": 15, "maxBeats": 5, "dialogueWordBudget": 35, "notes": ["..."] }, "technicalNotes": ["..."], "dialogueNotes": ["..."] } Rules: - Keep beats short and reusable. - Prefer 3-6 beats. - Capture the reusable mechanism, not copied claims or brand-specific language. - Include style layers for casting, setting, framing, lighting, pacing, and edit rhythm when visible. - Compress long references into a 15-second default unless the source duration clearly demands otherwise. - "keep", "change", and "reusableVariables" should be concise production notes. - Do not wrap the JSON in markdown fences.`; /** * True only for an existing, regular file. A directory path (which `existsSync` * alone would wave through to ffmpeg) returns false, as do missing paths and * unstattable sources, so only real files take the frame-ingest path. */ function isReadableFile(source: string): boolean { try { return statSync(source).isFile(); } catch { return false; } } function parseGeminiTextResponse(payload: unknown): string { const candidates = (payload as { candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }> }).candidates; const text = candidates?.[0]?.content?.parts ?.map((part) => part.text ?? '') .join('\n') .trim(); if (!text) { throw new Error('Gemini analyze response did not contain text output.'); } return text; } function parseAnalyzeJson(text: string): Omit { const cleaned = text.replace(/^```json\s*/i, '').replace(/^```\s*/i, '').replace(/\s*```$/i, '').trim(); let parsed: Partial; try { parsed = JSON.parse(cleaned) as Partial; } catch { throw new Error(`Gemini analyze response was not valid JSON. First 200 chars: ${cleaned.slice(0, 200)}`); } const beats = parsed.structure?.beats ?? []; if (!Array.isArray(beats) || beats.length === 0) { throw new Error('Gemini analyze response did not include structure.beats.'); } return { pacing: { label: parsed.pacing?.label ?? 'mixed', notes: parsed.pacing?.notes ?? [], }, structure: { ...(parsed.structure?.hook ? { hook: parsed.structure.hook } : {}), beats, ...(parsed.structure?.ending ? { ending: parsed.structure.ending } : {}), }, motionClassification: { primaryMode: parsed.motionClassification?.primaryMode ?? 'unknown', notes: parsed.motionClassification?.notes ?? [], }, keep: parsed.keep ?? [], change: parsed.change ?? [], reusableVariables: parsed.reusableVariables ?? [], ...(Array.isArray(parsed.styleLayers) ? { styleLayers: parsed.styleLayers } : {}), ...(parsed.beatCompression ? { beatCompression: parsed.beatCompression } : {}), ...(Array.isArray(parsed.technicalNotes) ? { technicalNotes: parsed.technicalNotes } : {}), ...(Array.isArray(parsed.dialogueNotes) ? { dialogueNotes: parsed.dialogueNotes } : {}), }; } export async function generateAnalyzeOutputWithGemini(input: { source: string; title?: string; durationSeconds?: number; endpoint?: string; fetcher?: typeof fetch; /** * Injectable frame-extraction seam (returns base64-encoded JPEG frames). * Defaults to the real ffmpeg helper; tests inject fakes so the suite never * shells out to ffmpeg. Only invoked when `source` is a readable local file. * The optional `durationSeconds` lets the extractor spread frames evenly * across the clip; injected fakes may ignore it. */ extractFrames?: (videoPath: string, durationSeconds?: number) => Promise; }): Promise { const endpoint = input.endpoint ?? process.env.VCLAW_GEMINI_API_ENDPOINT ?? DEFAULT_GEMINI_ANALYZE_ENDPOINT; // Only attempt frame ingestion when the source is a readable LOCAL FILE. // Directories (a `statSync().isFile()` rules them out), URLs, and missing // paths fall back to today's exact text-only behavior, and any extraction // failure (ffmpeg missing, corrupt file) also falls back — analyze must never // be worse than before. let frames: string[] = []; if (isReadableFile(input.source)) { const extractFrames = input.extractFrames ?? extractAnalyzeFrames; try { frames = await extractFrames(input.source, input.durationSeconds); } catch (error) { process.stderr.write( `[analyze/gemini] frame extraction failed (${(error as Error).message}); falling back to text-only analysis\n`, ); frames = []; } } const baseText = `${ANALYZE_PROMPT}\n\nSource: ${input.source}\nTitle: ${input.title ?? 'Untitled reference'}\nDuration: ${input.durationSeconds ?? 'unknown'} seconds`; const promptText = frames.length > 0 ? `${baseText}\n\nThe attached ${frames.length} image(s) are frames sampled evenly across the whole reference video (start to end), in order. Base your analysis on what they actually show.` : baseText; const parts = [ ...frames.map((data) => ({ inlineData: { mimeType: 'image/jpeg', data } })), { text: promptText }, ]; const response = await fetchGeminiWithPool( (key) => `${endpoint}${endpoint.includes('?') ? '&' : '?'}key=${encodeURIComponent(key)}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Connection': 'close', }, body: JSON.stringify({ contents: [{ parts, }], generationConfig: { temperature: 0.2, maxOutputTokens: 4096, responseMimeType: 'application/json', }, }), }, { fetcher: input.fetcher, onRetry: (label, status) => { process.stderr.write(`[analyze/gemini] ${label} returned HTTP ${status}; rotating key\n`); }, }, ); if (!response.ok) { const body = await response.text().catch(() => ''); throw new Error(`Gemini analyze request failed with HTTP ${response.status}: ${safeErrorBody(body)}`); } let payload: unknown; try { payload = await response.json(); } catch { throw new Error('Gemini analyze request returned a 2xx response with an unparseable JSON body.'); } const text = parseGeminiTextResponse(payload); const generated = parseAnalyzeJson(text); return createAnalyzeOutput({ reference: { source: input.source, ...(input.title ? { title: input.title } : {}), ...(input.durationSeconds !== undefined ? { durationSeconds: input.durationSeconds } : {}), }, ...generated, }); } function multiShotImageMimeType(imagePath: string): string { switch (extname(imagePath).toLowerCase()) { case '.jpg': case '.jpeg': return 'image/jpeg'; case '.webp': return 'image/webp'; case '.png': default: return 'image/png'; } } // Authors a multi-shot cinematic prompt body via Gemini, conditioned on the // actual reference image bytes (sent as an inlineData part). Requires a // configured Gemini key pool (fetchGeminiWithPool throws if the pool is empty). // The stub path (VCLAW_MULTISHOT_AUTO_STUB) is handled upstream in // generateMultiShotPromptText; this function is only called for the live path. export async function generateMultiShotWithGemini(input: { preset: import('./multi-shot-prompt.js').MultiShotPreset; imagePath: string; character?: string; action?: string; location: string; timeOfDay: string; repairInstructions?: string; /** Injectable fetch for tests; falls back to the global Gemini key pool. */ fetcher?: typeof fetch; }): Promise { const endpoint = process.env.VCLAW_GEMINI_API_ENDPOINT ?? DEFAULT_GEMINI_ANALYZE_ENDPOINT; const brief = [ `Preset: ${input.preset.name} (${input.preset.totalSeconds}s total, ${input.preset.minShotSeconds}-${input.preset.maxShotSeconds}s per shot, max ${input.preset.maxChars} chars)`, `Style: ${input.preset.styleLine}`, `Audio: ${input.preset.audioLine}`, `Location: ${input.location}, ${input.timeOfDay}`, ...(input.character ? [`Character: ${input.character}`] : []), ...(input.action ? [`Action: ${input.action}`] : []), ...(input.repairInstructions ? [`Repair required: ${input.repairInstructions}`] : []), ].join('\n'); const promptText = `You are a cinematographer authoring a compressed timecoded multi-shot prompt for an AI video generator, conditioned on the attached reference image.\n\nRules:\n- From the reference image, extract a compact 60-120 character visual description of the subject (hair, facial hair, skin tone, build, clothing, accessories, overall vibe) and weave those identifying details across the shots where they are visible (clothing in wider shots, face in close-ups); do not front-load them in one block\n- Use timecodes in [MM:SS - MM:SS] format, contiguous from 00:00 to ${String(Math.floor(input.preset.totalSeconds / 60)).padStart(2,'0')}:${String(input.preset.totalSeconds % 60).padStart(2,'0')}\n- Each shot: ${input.preset.minShotSeconds}-${input.preset.maxShotSeconds}s; vary shot size, lens, angle, movement shot-to-shot (never repeat consecutively)\n- End with three metadata lines: Location, Style, Audio\n- Total prompt under ${input.preset.maxChars} characters\n- Return ONLY the prompt body, no explanation\n\nBrief:\n${brief}`; // Read the real image bytes; on the live path a missing/unreadable file // surfaces as a propagated error. const imageBytes = await readFile(input.imagePath); const imageData = imageBytes.toString('base64'); const mimeType = multiShotImageMimeType(input.imagePath); const response = await fetchGeminiWithPool( (key) => `${endpoint}${endpoint.includes('?') ? '&' : '?'}key=${encodeURIComponent(key)}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Connection': 'close' }, body: JSON.stringify({ contents: [{ parts: [ { inlineData: { mimeType, data: imageData } }, { text: promptText }, ], }], generationConfig: { temperature: 0.7, maxOutputTokens: 800, responseMimeType: 'text/plain', }, }), }, { ...(input.fetcher ? { fetcher: input.fetcher } : {}), onRetry: (label, status) => { process.stderr.write(`[multi-shot/gemini] ${label} returned HTTP ${status}; rotating key\n`); }, }, ); if (!response.ok) { const body = await response.text().catch(() => ''); throw new Error(`Gemini multi-shot request failed with HTTP ${response.status}: ${safeErrorBody(body)}`); } let payload: unknown; try { payload = await response.json(); } catch { throw new Error('Gemini multi-shot request returned a 2xx response with an unparseable JSON body.'); } return parseGeminiTextResponse(payload).trim(); }