import { readFile } from 'node:fs/promises'; import { rewriteEmotionAsPhysical } from './emotion-cues.js'; import { CAMERA_MOVE_VOCABULARY, SHOT_TYPE_VOCABULARY, SHOT_SIZE_VOCABULARY, LENS_VOCABULARY, ANGLE_VOCABULARY, } from './prompt-quality.js'; import type { CategoryDescriptor } from './category-registry.js'; import { soundDesign } from './cinematography.js'; export interface MultiShotPreset { name: string; totalSeconds: number; minShotSeconds: number; maxShotSeconds: number; minShots: number; maxShots: number; maxChars: number; styleLine: string; audioLine: string; } export const CINEMATIC_15S_PRESET: MultiShotPreset = { name: 'cinematic-15s', totalSeconds: 15, minShotSeconds: 2, maxShotSeconds: 5, minShots: 3, maxShots: 7, maxChars: 1500, styleLine: 'Cool shadows, natural skin tones. IMAX-scale composition, deep focus, practical lighting. High contrast, grounded realism. In the style of a Christopher Nolan movie.', audioLine: 'Diegetic sound only — natural ambience, environmental foley, and subject-driven sound.', }; export const SEEDANCE_10S_PRESET: MultiShotPreset = { name: 'seedance-10s', totalSeconds: 10, minShotSeconds: 2, maxShotSeconds: 5, minShots: 2, maxShots: 5, maxChars: 1500, styleLine: CINEMATIC_15S_PRESET.styleLine, audioLine: CINEMATIC_15S_PRESET.audioLine, }; export const VEO_8S_PRESET: MultiShotPreset = { name: 'veo-8s', totalSeconds: 8, minShotSeconds: 2, maxShotSeconds: 4, minShots: 2, maxShots: 4, maxChars: 1500, styleLine: CINEMATIC_15S_PRESET.styleLine, audioLine: CINEMATIC_15S_PRESET.audioLine, }; export const RUNWAY_10S_PRESET: MultiShotPreset = { name: 'runway-10s', totalSeconds: 10, minShotSeconds: 2, maxShotSeconds: 5, minShots: 2, maxShots: 5, maxChars: 1000, styleLine: CINEMATIC_15S_PRESET.styleLine, audioLine: CINEMATIC_15S_PRESET.audioLine, }; export const MUSIC_VIDEO_PRESET: MultiShotPreset = { name: 'music-video-15s', totalSeconds: 15, minShotSeconds: 1, maxShotSeconds: 4, minShots: 4, maxShots: 12, maxChars: 1500, styleLine: 'Saturated stage color, rhythmic lighting, performance energy. Bold contrast, expressive grade, beat-driven cutting.', audioLine: 'Music-driven — the track sets the tempo and cuts land on the beat, with diegetic accents layered under the mix.', }; export const SOCIAL_HOOK_PRESET: MultiShotPreset = { name: 'social-hook-8s', totalSeconds: 8, minShotSeconds: 1, maxShotSeconds: 3, minShots: 3, maxShots: 8, maxChars: 1000, styleLine: 'Bright phone-native vertical look, punchy saturation, fast pattern-interrupt cutting, captiony hook energy.', audioLine: 'Punchy social mix — a hook SFX up front, trending-audio energy, and captiony beats driving fast cuts.', }; const PRESET_REGISTRY: ReadonlyMap = new Map([ [CINEMATIC_15S_PRESET.name, CINEMATIC_15S_PRESET], [SEEDANCE_10S_PRESET.name, SEEDANCE_10S_PRESET], [VEO_8S_PRESET.name, VEO_8S_PRESET], [RUNWAY_10S_PRESET.name, RUNWAY_10S_PRESET], [MUSIC_VIDEO_PRESET.name, MUSIC_VIDEO_PRESET], [SOCIAL_HOOK_PRESET.name, SOCIAL_HOOK_PRESET], ]); const GENRE_STYLE_LINES: ReadonlyMap = new Map([ ['music-video', 'Saturated stage color, rhythmic lighting, performance energy. Bold contrast, expressive grade, beat-driven cutting.'], ['action', 'Punchy high-contrast teal-and-orange, kinetic handheld, crushed shadows, aggressive coverage.'], ['anime', '2D anime cel-shading, clean line work, painterly backgrounds, vivid saturated palette.'], ['noir', 'High-contrast black and white, harsh chiaroscuro, deep shadow, 35mm grain.'], ['influencer', 'Bright clean social-first look, soft flattering key, natural skin, crisp and current.'], ['pixar', 'Stylized 3D render, soft global illumination, expressive proportions, warm inviting palette.'], ['cgi', 'Photoreal CGI render, ray-traced global illumination, physically-based materials, crisp specular highlights, volumetric light.'], ['cartoon', 'Hand-drawn 2D cartoon, bold clean outlines, flat cel fills, bouncy exaggerated motion, saturated primary palette.'], ['social', 'Bright phone-native vertical look, punchy saturation, fast pattern-interrupt cutting, captiony hook energy.'], ]); /** * Resolve a preset style line for a genre. Falls back to the cinematic Nolan * line (today's hardcoded default) for unknown/absent genres. */ export function resolveStyleLine(genre?: string): string { if (!genre) return CINEMATIC_15S_PRESET.styleLine; return GENRE_STYLE_LINES.get(genre.toLowerCase()) ?? CINEMATIC_15S_PRESET.styleLine; } export function knownPresetNames(): readonly string[] { return Array.from(PRESET_REGISTRY.keys()); } // Provider/route hint → preset, keyed on the provider FAMILY (the first token of // the hint, e.g. `seedance-direct` → `seedance`, `veo-useapi` → `veo`, // `google-flow` → `google`). Matching the family token rather than a substring // avoids misfires like `veo-via-runway-proxy` resolving to runway, and keeps the // mapping next to the preset registry it points at — the single source of truth. const PROVIDER_FAMILY_PRESET: ReadonlyMap = new Map([ ['seedance', SEEDANCE_10S_PRESET.name], ['veo', VEO_8S_PRESET.name], ['google', VEO_8S_PRESET.name], ['flow', VEO_8S_PRESET.name], ['runway', RUNWAY_10S_PRESET.name], ]); export function presetNameForProvider(hint: string | undefined): string | undefined { if (!hint) return undefined; const family = hint.trim().toLowerCase().split(/[-_:/\s]/)[0]; return PROVIDER_FAMILY_PRESET.get(family); } export function listMultiShotPresets(): readonly MultiShotPreset[] { return Array.from(PRESET_REGISTRY.values()); } export function resolvePreset(name?: string): MultiShotPreset { if (name === undefined) return CINEMATIC_15S_PRESET; const preset = PRESET_REGISTRY.get(name); if (!preset) { throw new Error( `unknown preset "${name}" (known: ${knownPresetNames().join(', ')})`, ); } return preset; } // Suggested camera-grid vocabularies. Shot sizes/angles/lenses are local to the // framework; prompt-quality's SHOT_TYPE_VOCABULARY is only re-exported for // consumers (it is not used when building the plan — SHOT_SIZES is). const SHOT_SIZES = SHOT_SIZE_VOCABULARY; const LENSES = LENS_VOCABULARY; const ANGLES = ANGLE_VOCABULARY; const MOVEMENTS = CAMERA_MOVE_VOCABULARY; export interface ShotSlot { index: number; start: number; end: number; timecode: string; shotSize: string; lens: string; angle: string; movement: string; } export interface ParsedMultiShotShot extends ShotSlot { description: string; } export interface ShotPlan { preset: MultiShotPreset; shots: ShotSlot[]; /** * The PRNG seed this plan was generated from. Always populated — when the * caller does not pass `--seed`, buildShotPlan picks one and records it here, * so the exact plan is reproducible from the emitted JSON: passing this value * back as `--seed` regenerates the identical shot sequence. This is the * determinism anchor for "exactly what you know you're going to get". */ seed: number; } export interface BuildShotPlanOptions { shots?: number; seed?: number; } // Deterministic, seedable PRNG so plans vary across calls but are reproducible in tests. function mulberry32(seed: number): () => number { let a = seed >>> 0; return () => { a |= 0; a = (a + 0x6d2b79f5) | 0; let t = Math.imul(a ^ (a >>> 15), 1 | a); t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; } export function formatTimecode(seconds: number): string { const mm = Math.floor(seconds / 60); const ss = seconds % 60; return `${String(mm).padStart(2, '0')}:${String(ss).padStart(2, '0')}`; } // Partition totalSeconds into `count` integer durations, each within [min, max]. function partitionDurations( total: number, count: number, min: number, max: number, rand: () => number, ): number[] { if (count * min > total || count * max < total) { throw new Error( `cannot partition ${total}s into ${count} shots within [${min}, ${max}]`, ); } const durations = new Array(count).fill(min); let remaining = total - count * min; const open = durations.map((_, i) => i); // indices still below max while (remaining > 0) { const pick = Math.floor(rand() * open.length); const i = open[pick]; durations[i] += 1; remaining -= 1; if (durations[i] >= max) open.splice(pick, 1); } return durations; } function pickNonRepeating(pool: readonly T[], prev: T | undefined, rand: () => number): T { if (pool.length === 1) return pool[0]; const candidates = prev === undefined ? pool : pool.filter((v) => v !== prev); return candidates[Math.floor(rand() * candidates.length)]; } export function buildShotPlan( preset: MultiShotPreset, options: BuildShotPlanOptions = {}, ): ShotPlan { // Resolve the seed once and record it on the returned plan so the exact // sequence is reproducible from disk (pass it back via --seed). const resolvedSeed = options.seed ?? Math.floor(Math.random() * 1e9); const rand = mulberry32(resolvedSeed); const arithMin = Math.ceil(preset.totalSeconds / preset.maxShotSeconds); const arithMax = Math.floor(preset.totalSeconds / preset.minShotSeconds); const minCount = Math.max(preset.minShots, arithMin); const maxCount = Math.min(preset.maxShots, arithMax); if (minCount > maxCount) { throw new Error( `preset "${preset.name}": shot-count window [${preset.minShots}, ${preset.maxShots}] cannot satisfy duration partition [${arithMin}, ${arithMax}]`, ); } let count = options.shots ?? minCount + Math.floor(rand() * (maxCount - minCount + 1)); // Clamp to [minCount, maxCount] so an explicit --shots stays feasible. if (count < minCount) count = minCount; if (count > maxCount) count = maxCount; const durations = partitionDurations( preset.totalSeconds, count, preset.minShotSeconds, preset.maxShotSeconds, rand, ); const shots: ShotSlot[] = []; let cursor = 0; let prevSize: string | undefined; let prevLens: string | undefined; let prevAngle: string | undefined; let prevMove: string | undefined; for (let i = 0; i < count; i += 1) { const start = cursor; const end = cursor + durations[i]; cursor = end; const shotSize = pickNonRepeating(SHOT_SIZES, prevSize, rand); const lens = pickNonRepeating(LENSES, prevLens, rand); const angle = pickNonRepeating(ANGLES, prevAngle, rand); const movement = pickNonRepeating(MOVEMENTS, prevMove, rand); prevSize = shotSize; prevLens = lens; prevAngle = angle; prevMove = movement; shots.push({ index: i, start, end, timecode: `[${formatTimecode(start)} - ${formatTimecode(end)}]`, shotSize, lens, angle, movement, }); } return { preset, shots, seed: resolvedSeed }; } export function assembleMetadataBlock( preset: MultiShotPreset, location: string, timeOfDay: string, genre?: string, ): string { const loc = timeOfDay ? `${location}, ${timeOfDay}` : location; const styleLine = genre !== undefined ? resolveStyleLine(genre) : preset.styleLine; return [ `Location: ${loc}`, `Style: ${styleLine}`, `Audio: ${preset.audioLine}`, ].join('\n'); } // Compose a full prompt body from a plan whose shots already carry `description`. export function composePromptText( plan: Array & { line: string }>, metadataBlock: string, ): string { const body = plan.map((s) => `${s.timecode} ${s.line}`).join('\n\n'); return `${body}\n\n${metadataBlock}`; } // Render a ShotPlan in Seedance's native prompt format: one flowing paragraph // with inline labeled segments (Style & Mood / Dynamic Description / Static // Description), a camera block in the existing per-shot emitter phrasing, and an // Audio footer. Pure and deterministic — it only reads `plan`/`descriptor`. export function composeSeedanceParagraph( plan: ShotPlan, descriptor: CategoryDescriptor, ): string { const { preset, shots } = plan; // Camera block: reuse the per-shot "shotSize, lens, angle, movement" phrasing, // collapsed onto one line so the paragraph stays a single block. const cameraBlock = shots .map((s) => `${s.shotSize}, ${s.lens}, ${s.angle}, ${s.movement}`) .join('; '); const motion = shots.map((s) => s.movement).join(', '); // Style & Mood is driven by the category's genre so a category actually looks // like its genre (e.g. cgi/cartoon/anime), not the generic Nolan preset line. // live-action genres have no GENRE_STYLE_LINES entry, so resolveStyleLine // returns the same Nolan line as the preset — those categories stay byte-stable. const styleMood = `${descriptor.label} — ${resolveStyleLine(descriptor.genre)}`; const dynamic = `the ${descriptor.subjectType} carries the action across ${shots.length} continuous beats (${motion})`; const staticScene = `${descriptor.label} scene, ${descriptor.genre} look, beat structure ${descriptor.beatTemplate}`; const segments = [ `Style & Mood: ${styleMood}`, `Dynamic Description: ${dynamic}`, `Static Description: ${staticScene}`, `Camera: ${cameraBlock}`, `Audio: ${preset.audioLine}`, ]; // Single space joins keep this one flowing paragraph (no "\n\n" block breaks). return segments.join(' '); } // Render a ShotPlan as a structured per-shot video-prompt layout: one block per // shot headed `SHOT `, followed by labeled lines (Framing / Scene / // Dialogue / SFX / Camera), closed by a single Audio footer from the preset. // Blocks are separated by blank lines. Pure and deterministic — it only reads // `plan`/`descriptor`. ShotSlot carries no per-shot name/scene/dialogue/sfx // fields, so those are derived deterministically (NAME falls back to the shot // size, then `Shot `; Dialogue/SFX render an em-dash placeholder when absent). export function composePerShotFormat( plan: ShotPlan, descriptor: CategoryDescriptor, ): string { const { preset, shots } = plan; const blocks = shots.map((s, i) => { const n = i + 1; const name = s.shotSize || `Shot ${n}`; const framing = `${s.shotSize}, ${s.angle}, ${s.movement}, ${s.lens}`; const scene = `${descriptor.label} — ${descriptor.subjectType} carries beat ${n} of ${shots.length} (${s.movement}); ${descriptor.genre} look, ${descriptor.beatTemplate} structure`; return [ `SHOT ${n} — ${name}`, `Framing: ${framing}`, `Scene: ${scene}`, `Dialogue: —`, `SFX: ${soundDesign(descriptor.audioProfile)}`, `Camera: ${s.movement}`, ].join('\n'); }); return `${blocks.join('\n\n')}\n\nAudio: ${preset.audioLine}`; } export interface DialogueLine { speaker: string; line: string; emotion?: string; secondSpeaker?: { speaker: string; line: string; emotion?: string }; } // Append spoken dialogue to a shot line using a clean two-speaker convention. // The first speaker gets a " says[, ]:" opener; a second // speaker (when present) gets exactly one " replies[, ]:" // opener. Pure and deterministic — no randomness or clock reads. export function withDialogue( shotLine: string, dialogue: DialogueLine, options?: { emotionCues?: boolean }, ): string { const rw = (emotion: string | undefined): string | undefined => options?.emotionCues ? rewriteEmotionAsPhysical(emotion) : emotion; const firstEmotion = rw(dialogue.emotion); const firstOpener = firstEmotion ? `${dialogue.speaker} says, ${firstEmotion}:` : `${dialogue.speaker} says:`; const segments = [shotLine, `${firstOpener} "${dialogue.line}"`]; if (dialogue.secondSpeaker) { const { speaker, line, emotion } = dialogue.secondSpeaker; const secondEmotion = rw(emotion); const replyOpener = secondEmotion ? `${speaker} replies, ${secondEmotion}:` : `${speaker} replies:`; segments.push(`${replyOpener} "${line}"`); } return segments.join(' '); } // Parse a CLI `--dialogue` value into a DialogueLine. The format is // `": "`, optionally with a second speaker after a `||` // separator: `"A: hi || B: bye"`. Each side must contain a colon; the first // colon splits speaker from line. THROWS on a malformed side (no colon) — there // is no silent fallback. Pure and deterministic. export function parseDialogueLine(value: string): DialogueLine { const parseSide = (raw: string): { speaker: string; line: string; emotion?: string } => { const idx = raw.indexOf(':'); if (idx === -1) { throw new Error(`malformed dialogue (expected ": "): ${raw.trim()}`); } const speaker = raw.slice(0, idx).trim(); let line = raw.slice(idx + 1).trim(); // Optional trailing [emotion] annotation, e.g. `Mara: It is fine. [scared]`. let emotion: string | undefined; const emotionMatch = line.match(/\s*\[([^\]]+)\]\s*$/); if (emotionMatch) { emotion = emotionMatch[1].trim(); line = line.slice(0, emotionMatch.index).trim(); } if (!speaker || !line) { throw new Error(`malformed dialogue (expected ": "): ${raw.trim()}`); } return { speaker, line, ...(emotion ? { emotion } : {}) }; }; const [firstRaw, ...rest] = value.split('||'); const first = parseSide(firstRaw); const dialogue: DialogueLine = { speaker: first.speaker, line: first.line, ...(first.emotion ? { emotion: first.emotion } : {}), }; if (rest.length > 0) { const second = parseSide(rest.join('||')); dialogue.secondSpeaker = { speaker: second.speaker, line: second.line, ...(second.emotion ? { emotion: second.emotion } : {}), }; } return dialogue; } export type Lang = 'en' | 'zh' | 'en+zh'; // Wrap a composed prompt block for bilingual delivery. `en` returns the English // text in a single fenced code block; `zh` returns ONE fenced block of the // translated text; `en+zh` returns TWO labeled fenced blocks (EN then 中文). // // Translation is injected via `opts.translate` so this stays pure/deterministic // and offline — the default is an identity passthrough. The contract for any // real translator is that numeric/technical specs (ft/s, Kelvin, °, %, dB) pass // through unchanged; with the identity default the ZH block is byte-identical to // the EN block, so spec tokens are trivially preserved. Pure and deterministic. export function composeBilingual( text: string, lang: Lang, opts: { translate?: (text: string) => string } = {}, ): string { const translate = opts.translate ?? ((t: string) => t); const fence = (body: string): string => `\`\`\`\n${body}\n\`\`\``; if (lang === 'en') return fence(text); if (lang === 'zh') return fence(translate(text)); return [`EN`, fence(text), `中文`, fence(translate(text))].join('\n'); } export { SHOT_SIZES, LENSES, ANGLES, MOVEMENTS, SHOT_TYPE_VOCABULARY }; let stubSequenceIndex = 0; // Brackets stay required (anchors the match so prose containing a "12:30" time // isn't parsed as a shot), but accept ASCII hyphen / en-dash / em-dash and // 1-2 digit minutes so a valid Gemini prompt isn't silently parsed to zero shots. const TIMECODE_LINE_RE = /^\s*\[(\d{1,2}):(\d{2})\s*[-–—]\s*(\d{1,2}):(\d{2})\]\s*(.*)$/; function secondsFromParts(mm: string, ss: string): number { return Number(mm) * 60 + Number(ss); } function findCanonicalTerm(haystack: string, pool: readonly string[]): string { const normalizedHaystack = haystack.toLowerCase().replace(/-/g, ' '); for (const term of [...pool].sort((a, b) => b.length - a.length)) { const normalizedTerm = term.toLowerCase().replace(/-/g, ' '); if (normalizedHaystack.includes(normalizedTerm)) return term; } return ''; } function stripShotLead(text: string): string { const dashIndex = text.search(/\s[—–-]\s/); if (dashIndex >= 0) { return text.slice(dashIndex + 3).trim(); } return text .replace(/^(?:[^,.]+,\s*){1,4}/, '') .replace(/^[:.]\s*/, '') .trim(); } export function parseMultiShotPrompt(promptText: string): ParsedMultiShotShot[] { const shots: ParsedMultiShotShot[] = []; const lines = promptText.split('\n').filter((line) => line.trim().length > 0); for (const line of lines) { const match = TIMECODE_LINE_RE.exec(line); if (!match) continue; const start = secondsFromParts(match[1], match[2]); const end = secondsFromParts(match[3], match[4]); const body = match[5].trim(); shots.push({ index: shots.length, start, end, timecode: `[${formatTimecode(start)} - ${formatTimecode(end)}]`, shotSize: findCanonicalTerm(body, SHOT_SIZES), lens: findCanonicalTerm(body, LENSES), angle: findCanonicalTerm(body, ANGLES), movement: findCanonicalTerm(body, MOVEMENTS), description: stripShotLead(body), }); } return shots; } // Authors a finished prompt body. When VCLAW_MULTISHOT_AUTO_STUB points to a file, // its contents are returned verbatim (test/offline path). Otherwise calls Gemini. export async function generateMultiShotPromptText(input: { preset: MultiShotPreset; imagePath: string; character?: string; action?: string; location: string; timeOfDay: string; repairInstructions?: string; }): Promise { const stub = process.env.VCLAW_MULTISHOT_AUTO_STUB; if (stub) { const raw = (await readFile(stub, 'utf-8')).trim(); try { const parsed = JSON.parse(raw); if (Array.isArray(parsed) && parsed.every((item) => typeof item === 'string')) { // A call with no repairInstructions is the first attempt of a fresh // generation sequence, so reset the cursor. Retries (repairInstructions // present) advance through the array. This keeps the module-global index // from bleeding across independent in-process generations. if (!input.repairInstructions) stubSequenceIndex = 0; const item = parsed[Math.min(stubSequenceIndex, parsed.length - 1)]; stubSequenceIndex += 1; return item.trim(); } } catch { // Plain-text stubs remain the default offline path. } return raw; } // Real path: delegate to the shared Gemini analyze plumbing. Dynamic import so // the Gemini module is only loaded on the live path (avoids a static import edge). const { generateMultiShotWithGemini } = await import('./gemini-analyze.js'); return generateMultiShotWithGemini({ preset: input.preset, imagePath: input.imagePath, character: input.character, action: input.action, location: input.location, timeOfDay: input.timeOfDay, repairInstructions: input.repairInstructions, }); }