/** * Voice clones — the "blank video with audio" voice-reference trick. * * Source of truth: the Jack-Vs-AI "cartoon show" workflow (VOICE DESIGN chapter). * The production-learned insight is that supplying a target voice to a video model * (Seedance 2 / Veo) as a raw MP3/WAV audio reference does NOT lock the voice — the * generation drifts to a generic accent. Supplying the SAME audio as the track of a * **black-frame video** ("blank video with the audio track there") DOES lock it: the * model treats it as a video reference and clones the voice as a driving asset. * * This module is the deterministic CLI half: it BUILDS that black-frame-plus-audio * MP4 from an audio sample (via the shared `runFfmpeg` helper) and persists each clip * as a reusable **voice clone** asset (`artifacts/voice-clones.json`), mirroring the * seedance-asset-library / environment-assets pattern. A voice clone can be bound to a * character so that any scene featuring that character auto-injects the blank video * into the Seedance `reference_videos` set at execution time — or reference a voice * explicitly with an `@` tag (see `execution-runtime.ts` / * `asset-tag-lookup.ts`). Absent artifact → byte-identical to today. * * Voice-drift fix from the workflow: `sliceSeconds` chops one long recording into N * short clips (the creator recommends 2-second slices per dialogue line) so the model * has cleaner per-line references. * * PURE / TESTABLE: `planBlankVideoWithAudio()` returns the ffmpeg argv WITHOUT * spawning, so the whole planning surface is offline-testable. The executor only * shells out when `dryRun` is false — and `runFfmpeg`/`ffprobeDuration` themselves * short-circuit on dryRun, so tests never need ffmpeg installed. */ import { existsSync } from 'node:fs'; import { mkdir, readFile } from 'node:fs/promises'; import { dirname, extname, join } from 'node:path'; import { resolveProjectWorkspace } from './workspace.js'; import { writeTextFileAtomic } from './atomic-write.js'; import { runFfmpeg, ffprobeDuration, STANDARD_VIDEO_ARGS, STANDARD_AUDIO_ARGS } from './assemble/ffmpeg.js'; import type { AssetTagEntry } from './prompt-rules.js'; // ── Artifact ────────────────────────────────────────────────────────────────── /** One registered voice clone in `voice-clones.json`. */ export interface VoiceCloneEntry { /** Voice-asset name (what an @Name tag / character binding resolves to). */ name: string; /** Optional character this voice belongs to (so the character always speaks it). */ character?: string; /** The source audio sample this clone was built from. */ sourceAudio: string; /** The built blank-video-with-audio clip (the reference the model consumes). */ clipPath: string; /** * Durable public URL for `clipPath`, hosted on Go Bananas. Required for the * seedance-direct r2v voice-lock (the remote API needs a hosted URL, not a local * path). Populated by `voice-clone --execute` when GB media hosting is available; * absent when hosting is unavailable (the clip stays a local path). */ hostedUrl?: string; /** When sliced for drift control: the per-line clip paths (in order). */ slices?: string[]; /** Optional human description of the voice character/tone. */ description?: string; /** Duration of the source audio in milliseconds (0 when not probed / dry-run). */ durationMs?: number; } export interface VoiceClonesArtifact { schemaVersion: 1; projectSlug: string; generatedAt: string; voices: VoiceCloneEntry[]; } export interface VoiceClonesLookup { voices: VoiceCloneEntry[]; /** name (lowercased) -> entry. */ voiceByName: Map; /** character (lowercased) -> entry, for character→voice binding. */ voiceByCharacter: Map; /** * name (lowercased) -> AssetTagEntry, ready for buildAssetTagLookup's * `voicesByName`. The referencePath is the blank-video clip so it classifies * into Seedance `reference_videos`. */ voiceEntryByName: Map; } export function voiceClonesPathFor(root: string, slug: string): string { return join(resolveProjectWorkspace(slug, root).artifactsDir, 'voice-clones.json'); } /** Directory the built blank-video clips live in (under the project). */ export function voiceCloneClipsDir(root: string, slug: string): string { return join(resolveProjectWorkspace(slug, root).projectDir, 'voices'); } export async function writeVoiceClones( root: string, slug: string, artifact: VoiceClonesArtifact, ): Promise { const path = voiceClonesPathFor(root, slug); await mkdir(dirname(path), { recursive: true }); await writeTextFileAtomic(path, `${JSON.stringify(artifact, null, 2)}\n`); } /** * Read `artifacts/voice-clones.json`. Returns empty lookups when absent (graceful — * the execution layer then injects no voice references, byte-identical to today). A * present-but-malformed file is NOT swallowed (JSON.parse surfaces it). */ export async function readVoiceClones(root: string, slug: string): Promise { const path = voiceClonesPathFor(root, slug); if (!existsSync(path)) { return { voices: [], voiceByName: new Map(), voiceByCharacter: new Map(), voiceEntryByName: new Map(), }; } const parsed = JSON.parse(await readFile(path, 'utf-8')) as Partial; const voices: VoiceCloneEntry[] = Array.isArray(parsed.voices) ? parsed.voices.filter((v): v is VoiceCloneEntry => !!v && !!v.name && !!v.clipPath) : []; const voiceByName = new Map(); const voiceByCharacter = new Map(); const voiceEntryByName = new Map(); for (const v of voices) { voiceByName.set(v.name.toLowerCase(), v); if (v.character) voiceByCharacter.set(v.character.toLowerCase(), v); voiceEntryByName.set(v.name.toLowerCase(), { descriptor: v.description ?? `the voice of ${v.character ?? v.name}`, // Prefer the durable hosted URL (required by remote routes); fall back to the // local clip when hosting was unavailable at clone time. referencePath: v.hostedUrl ?? v.clipPath, }); } return { voices, voiceByName, voiceByCharacter, voiceEntryByName }; } // ── Blank-video-with-audio builder ────────────────────────────────────────────── const AUDIO_EXTENSIONS = new Set(['.mp3', '.wav', '.m4a', '.aac', '.ogg', '.flac']); export interface BlankVideoPlanOptions { /** Source audio sample path. */ audioPath: string; /** Output clip path (.mp4). */ outputPath: string; /** Black-frame canvas width (default 1280). */ width?: number; /** Black-frame canvas height (default 720). */ height?: number; /** Optional fixed slice window for drift control (e.g. 2 seconds). */ startSeconds?: number; durationSeconds?: number; /** * Clamp the OUTPUT to this many seconds (output-side `-t`). x264 keeps encoding * frames after the audio ends, so `-shortest` alone lets the video stream overshoot * the audio (e.g. 14s audio -> 16.2s video) and trip the r2v duration cap. Setting * this to the audio length forces the whole clip to exactly that duration. */ clampSeconds?: number; } /** * Build the ffmpeg argv for one black-frame-plus-audio clip WITHOUT spawning. * * `-f lavfi -i color=c=black:s=WxH:r=24` synthesizes the black video; the audio * sample is the second input. When a slice window is given, `-ss -t ` * trims the AUDIO input. `clampSeconds` adds an output-side `-t` that forces the whole * clip to that length — necessary because `-shortest` alone lets x264 overshoot the * audio (trailing video frames), which would trip the r2v duration cap. PURE — splice * the result straight into a runFfmpeg call. Throws on a non-audio source (code 1). */ export function planBlankVideoWithAudio(opts: BlankVideoPlanOptions): string[] { const ext = extname(opts.audioPath).toLowerCase(); if (!AUDIO_EXTENSIONS.has(ext)) { throw new Error( `voice-clone source must be an audio file (${[...AUDIO_EXTENSIONS].join(', ')}); got "${opts.audioPath}".`, ); } const width = opts.width ?? 1280; const height = opts.height ?? 720; const args: string[] = [ '-f', 'lavfi', '-i', `color=c=black:s=${width}x${height}:r=24`, ]; // Trim the AUDIO input window when slicing (apply -ss/-t to the audio input). if (opts.startSeconds !== undefined) args.push('-ss', String(opts.startSeconds)); if (opts.durationSeconds !== undefined) args.push('-t', String(opts.durationSeconds)); args.push( '-i', opts.audioPath, ...STANDARD_VIDEO_ARGS, ...STANDARD_AUDIO_ARGS, // Map the synthesized video + the real audio. '-map', '0:v:0', '-map', '1:a:0', ); // Output-side clamp: forces the whole clip (video + audio) to clampSeconds, so the // video can't overshoot the audio. Falls back to -shortest when no clamp is known // (e.g. dry-run, where the audio length isn't probed). if (opts.clampSeconds !== undefined) args.push('-t', String(opts.clampSeconds)); args.push('-shortest', opts.outputPath); return args; } export interface VoiceCloneInput { /** Voice-asset name. */ name: string; /** Source audio sample path. */ audioPath: string; /** Optional character to bind the voice to. */ character?: string; /** Optional voice description. */ description?: string; /** Slice the recording into windows of this many seconds (drift fix; 0/undefined = single clip). */ sliceSeconds?: number; /** Black-frame canvas. */ width?: number; height?: number; } /** * Max reference-video length the Seedance/Dreamina r2v voice-lock accepts * (`dreamina-seedance-2-0` rejects refs over this with HTTP 500). A longer voice * clip is warned about at clone time so the operator slices it before it hits the * provider as an opaque 500. */ export const VOICE_REF_MAX_SECONDS = 15.2; export interface BuildVoiceCloneOptions { workspaceRoot: string; slug: string; /** ISO timestamp (injected so the artifact stays deterministic/testable). */ generatedAt: string; /** When true, plan only: return the entry + ffmpeg commands but spawn nothing. */ dryRun?: boolean; /** Override the ffmpeg binary (tests / non-standard installs). */ ffmpegBin?: string; /** * Host the built clip and return its durable URL (or null when unavailable). * Injected so it is testable and so dry-runs never touch the network; the handler * wires the real Go Bananas uploader. Omitted/undefined -> no hosting attempted. */ hostMedia?: (clipPath: string) => Promise; } export interface BuildVoiceCloneResult { entry: VoiceCloneEntry; /** The ffmpeg command line(s) that were (or, on dryRun, would be) run. */ commands: string[]; /** Advisory warnings (duration over the r2v cap, hosting unavailable, …). */ warnings: string[]; } /** * Build a single voice clone: render its blank-video-with-audio clip(s) and return * the artifact entry. On `dryRun`, no ffmpeg is spawned (commands are still * returned for inspection) and durationMs is 0. * * Slicing: when `sliceSeconds > 0`, the source is probed for its length and one * `-NN.mp4` clip is emitted per window; `clipPath` points at the first slice * (the primary reference) and `slices[]` lists them all. Without slicing, a single * `.mp4` clip is built. */ export async function buildVoiceClone( input: VoiceCloneInput, opts: BuildVoiceCloneOptions, ): Promise { if (!input.name?.trim()) throw new Error('voice-clone requires a --name.'); if (!input.audioPath?.trim()) throw new Error('voice-clone requires an --audio sample.'); if (!opts.dryRun && !existsSync(input.audioPath)) { throw new Error(`voice-clone --audio not found: ${input.audioPath}`); } // Validate the source extension up front — before any ffprobe — so a non-audio // sample fails fast with the input-error message (planBlankVideoWithAudio also // checks, but now that we probe duration first, the early check keeps the error // classification stable: input error / exit 1, not an ffprobe failure / exit 2). const sourceExt = extname(input.audioPath).toLowerCase(); if (!AUDIO_EXTENSIONS.has(sourceExt)) { throw new Error( `voice-clone source must be an audio file (${[...AUDIO_EXTENSIONS].join(', ')}); got "${input.audioPath}".`, ); } const clipsDir = voiceCloneClipsDir(opts.workspaceRoot, opts.slug); if (!opts.dryRun) await mkdir(clipsDir, { recursive: true }); const safeName = input.name .trim() .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') || 'voice'; const commands: string[] = []; const slices: string[] = []; const warnings: string[] = []; let durationMs = 0; const sliceSeconds = input.sliceSeconds && input.sliceSeconds > 0 ? input.sliceSeconds : 0; if (sliceSeconds > 0) { durationMs = await ffprobeDuration(input.audioPath, { dryRun: opts.dryRun }); // On dry-run we cannot probe, so plan a single representative slice; the real // run probes and emits the full set. Either way the arg shapes are identical. const totalSeconds = durationMs > 0 ? durationMs / 1000 : sliceSeconds; const count = Math.max(1, Math.ceil(totalSeconds / sliceSeconds)); for (let i = 0; i < count; i += 1) { const outputPath = join(clipsDir, `${safeName}-${String(i + 1).padStart(2, '0')}.mp4`); const args = planBlankVideoWithAudio({ audioPath: input.audioPath, outputPath, ...(input.width !== undefined ? { width: input.width } : {}), ...(input.height !== undefined ? { height: input.height } : {}), startSeconds: i * sliceSeconds, durationSeconds: sliceSeconds, clampSeconds: sliceSeconds, }); const { command } = await runFfmpeg(args, { dryRun: opts.dryRun, ...(opts.ffmpegBin ? { ffmpegBin: opts.ffmpegBin } : {}), }); commands.push(command); slices.push(outputPath); } } else { const outputPath = join(clipsDir, `${safeName}.mp4`); // Probe the audio length BEFORE building so we can clamp the output to it (the // video would otherwise overshoot — see planBlankVideoWithAudio). 0 on dry-run. if (!opts.dryRun) durationMs = await ffprobeDuration(input.audioPath, { dryRun: false }); const args = planBlankVideoWithAudio({ audioPath: input.audioPath, outputPath, ...(input.width !== undefined ? { width: input.width } : {}), ...(input.height !== undefined ? { height: input.height } : {}), ...(durationMs > 0 ? { clampSeconds: durationMs / 1000 } : {}), }); const { command } = await runFfmpeg(args, { dryRun: opts.dryRun, ...(opts.ffmpegBin ? { ffmpegBin: opts.ffmpegBin } : {}), }); commands.push(command); slices.push(outputPath); } // Duration preflight (1B): a single voice reference longer than the r2v cap is // rejected by Seedance with an opaque HTTP 500. Warn (without slicing) so the // operator slices it; a sliced clone's primary slice is already short, so only // warn for the un-sliced case. if (sliceSeconds === 0 && durationMs > 0 && durationMs / 1000 > VOICE_REF_MAX_SECONDS) { warnings.push( `voice clip is ${(durationMs / 1000).toFixed(1)}s; Seedance r2v requires the reference video ≤${VOICE_REF_MAX_SECONDS}s. ` + `Re-run with --slice-seconds ${Math.floor(VOICE_REF_MAX_SECONDS)} (or a shorter sample) before using it on seedance-direct.`, ); } // Host the primary clip so remote routes get a durable public URL (seedance-direct // r2v needs a hosted ref, not a local path). Injected + executor-only; on dry-run // or when hosting is unavailable, hostedUrl stays unset and the clip stays local. let hostedUrl: string | undefined; if (!opts.dryRun && opts.hostMedia) { const url = await opts.hostMedia(slices[0]); if (url) hostedUrl = url; else warnings.push('voice clip not hosted; remote routes (seedance-direct) need a hosted URL — host it before use.'); } const entry: VoiceCloneEntry = { name: input.name.trim(), ...(input.character ? { character: input.character.trim() } : {}), sourceAudio: input.audioPath, clipPath: slices[0], ...(hostedUrl ? { hostedUrl } : {}), ...(slices.length > 1 ? { slices } : {}), ...(input.description ? { description: input.description.trim() } : {}), durationMs, }; return { entry, commands, warnings }; } /** * Build one voice clone and merge it into `voice-clones.json` (replacing any entry * with the same name). Returns the persisted artifact + the build result. On * `dryRun`, nothing is written and nothing is spawned — the would-be artifact is * returned for inspection. */ export async function registerVoiceClone( input: VoiceCloneInput, opts: BuildVoiceCloneOptions, ): Promise<{ artifact: VoiceClonesArtifact; result: BuildVoiceCloneResult }> { const result = await buildVoiceClone(input, opts); const existing = await readVoiceClones(opts.workspaceRoot, opts.slug); const merged = existing.voices.filter( (v) => v.name.toLowerCase() !== result.entry.name.toLowerCase(), ); merged.push(result.entry); const artifact: VoiceClonesArtifact = { schemaVersion: 1, projectSlug: opts.slug, generatedAt: opts.generatedAt, voices: merged, }; if (!opts.dryRun) { await writeVoiceClones(opts.workspaceRoot, opts.slug, artifact); } return { artifact, result }; }