/** * Motion-overlay execution (Phase 4) — V2V + audio-restore + stitch. * * `runMotionOverlay` drives the render half of `vclaw video motion-overlay * --execute`. The PLAN (a `MotionOverlayPlan` from `run.ts`) stays the source of * truth: this module renders each of its takes IN ORDER through three steps and * stitches the results into the final reel. It re-derives no slicing, prompting, * or routing — the plan already fixed those. * * Per take: * 1. Omni Flash V2V — feed the take's base footage (`takes/…mp4`) + the take's * composed prompt to the Google Flow omni-flash video-to-video transport. * Flow paints the kinetic typography / icon / metaphor overlay while keeping * the spoken content, but it RE-ENCODES the audio. * 2. Audio-restore — `ffmpeg -i -i -map 0:v -map 1:a -c:v copy * -c:a aac `: take the annotated VIDEO from the V2V output and the * ORIGINAL AUDIO from the source take, so the reel honors CRITICAL RULE 1 * (pass the original voiceover through) as closely as the transport allows. * 3. Collect the audio-restored take. * Then the collected takes are clip-stitched (in take order) into the final reel. * * Credit safety is structural: every side effect (V2V submit/poll, the * audio-restore ffmpeg mux, and the stitch) is delegated to an injectable * {@link MotionOverlayStepRunner}. The CLI supplies a runner that shells the real * `vclaw video` V2V path + ffmpeg + the assemble clip-stitch; tests inject a fake * runner, so the whole orchestrator runs offline with NO provider spend. * * Dry vs execute: default (`confirmSpend` falsey) is a dry plan — the runner is * NOT invoked at all and the report enumerates the steps that WOULD run. Only * `confirmSpend: true` actually drives the runner. This mirrors `studio/execute.ts`. */ import { mkdir } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { runFfmpeg } from '../assemble/ffmpeg.js'; import { stitch } from '../assemble/stitch.js'; import { takeStem } from './plan.js'; import type { MotionOverlayPlan, MotionTake, TakeSplit } from './types.js'; export class MotionOverlayExecuteError extends Error { constructor( public readonly code: string, message: string, ) { super(message); this.name = 'MotionOverlayExecuteError'; } } /** Inputs for a single take's V2V render. */ export interface MotionOverlayV2VStep { index: number; /** Absolute path to the take's base footage (the frame-accurate cut). */ baseFootage: string; /** The full composed prompt for this take. */ prompt: string; /** Absolute path the annotated (audio re-encoded) take should be written to. */ outputPath: string; } /** Inputs for a single take's audio-restore mux. */ export interface MotionOverlayAudioRestoreStep { index: number; /** Absolute path to the V2V output (annotated video, re-encoded audio). */ v2vVideo: string; /** Absolute path to the original take (the source of the audio to restore). */ originalTake: string; /** Absolute path the audio-restored take should be written to. */ outputPath: string; } /** Inputs for the final clip-stitch over the audio-restored takes. */ export interface MotionOverlayStitchStep { /** Ordered absolute paths of the audio-restored takes (take order). */ takeClips: string[]; /** Absolute path the final reel should be written to. */ outputPath: string; } /** * Injectable step runner — the single seam through which ALL side effects flow. * The CLI supplies a runner that shells the real `vclaw video` omni-flash V2V * path, the audio-restore ffmpeg mux, and the assemble clip-stitch. Tests inject * a fake so the orchestrator runs offline with no spend. */ export interface MotionOverlayStepRunner { /** Render one take through omni-flash V2V. Resolves once the annotated take exists. */ runV2V(step: MotionOverlayV2VStep): Promise; /** * Restore the original take audio onto the V2V output via the ffmpeg mux * `-map 0:v -map 1:a -c:v copy -c:a aac`. Resolves once the muxed take exists. */ restoreAudio(step: MotionOverlayAudioRestoreStep): Promise; /** Clip-stitch the audio-restored takes (in order) into the final reel. */ stitch(step: MotionOverlayStitchStep): Promise; } export interface RunMotionOverlayOptions { /** The injectable step runner (CLI shells real path; tests inject a fake). */ runner: MotionOverlayStepRunner; /** * Permit the credit-spending V2V + ffmpeg + stitch steps to RUN. Default false * → dry plan (the runner is never invoked); the report enumerates the steps. */ confirmSpend?: boolean; /** * Avatar-host layout only: a per-take map (take index → absolute host base clip * path) produced by `avatar-host.runAvatarHostGeneration`. When present for a * take, the V2V step uses the HOST clip as its base footage instead of the * source take — but the audio-restore step still pulls the ORIGINAL take's * audio (the host visual is a stand-in; the real voiceover is preserved). Absent * (or no entry for a take) → the source take is the V2V base, as for every other * layout. */ hostBaseByIndex?: Record; } /** What happened (or would happen) to one take during execution. */ export interface MotionOverlayTakeResult { index: number; anatomy: MotionTake['anatomy']; /** * Absolute path to the V2V base footage. The source take for every layout * EXCEPT avatar-host, where it is the generated host base clip. */ baseFootage: string; /** * Absolute path to the take whose audio is restored onto the V2V output. Always * the ORIGINAL source take (the real voiceover), even on avatar-host. */ audioSource: string; /** Absolute path to the V2V (annotated) output. */ v2vVideo: string; /** Absolute path to the audio-restored take (what feeds the stitch). */ restoredTake: string; /** ran = the runner was invoked; planned = dry (not invoked). */ status: 'ran' | 'planned'; } export type MotionOverlayExecuteMode = 'dry' | 'execute'; export interface MotionOverlayExecuteReport { schemaVersion: 1; mode: MotionOverlayExecuteMode; /** True only when the runner actually rendered (confirmSpend). */ executed: boolean; takes: MotionOverlayTakeResult[]; /** Absolute path to the final stitched reel. */ reelPath: string; /** Set in dry mode: how to render for real. */ hint?: string; } /** The subdir under the work folder where V2V outputs land. */ export const V2V_DIRNAME = 'v2v'; /** The subdir under the work folder where audio-restored takes land. */ export const RESTORED_DIRNAME = 'restored'; /** The final reel filename written into the work folder. */ export const REEL_FILENAME = 'motion-overlay-reel.mp4'; /** Resolve the per-take base-footage path from a take's manifest `file` field. */ function baseFootagePath(workdir: string, take: MotionTake): string { // `take.file` is the work-folder-relative `takes/…mp4` path written by plan.ts. return join(workdir, take.file); } /** * Reconstruct a take's stem (e.g. `take-01_0s-10s`) from its manifest fields so * the V2V / restored outputs sit beside the takes with matching names. We rebuild * the same {@link takeStem} the planner used from the take's index + bounds. */ function takeStemOf(take: MotionTake): string { const split: TakeSplit = { index: take.index, start: take.start, end: take.end, segmentIndices: [] }; return takeStem(split); } /** * Execute (or dry-plan) a motion-overlay reel from its plan. * * Takes are processed strictly IN ORDER; the final reel stitches the * audio-restored takes in that same order. Pure control flow — all I/O is * delegated to `opts.runner`. Never mutates the input plan. * * Throws {@link MotionOverlayExecuteError} (`motion_overlay_no_takes`) if the * plan has zero takes (nothing to render or stitch). */ export async function runMotionOverlay( plan: MotionOverlayPlan, opts: RunMotionOverlayOptions, ): Promise { if (plan.takes.length === 0) { throw new MotionOverlayExecuteError( 'motion_overlay_no_takes', 'motion-overlay execute: the plan has no takes to render.', ); } const confirmSpend = opts.confirmSpend === true; const mode: MotionOverlayExecuteMode = confirmSpend ? 'execute' : 'dry'; const reelPath = join(plan.workdir, REEL_FILENAME); const results: MotionOverlayTakeResult[] = []; // Takes are sorted by index defensively so the stitch order is deterministic // regardless of manifest ordering. const orderedTakes = [...plan.takes].sort((a, b) => a.index - b.index); const hostBaseByIndex = opts.hostBaseByIndex ?? {}; for (const take of orderedTakes) { const stem = takeStemOf(take); // The ORIGINAL source take — always the audio source (the real voiceover). const sourceTake = baseFootagePath(plan.workdir, take); // The V2V base footage: the generated host clip on avatar-host (when present), // else the source take. The audio-restore source stays the source take. const baseFootage = hostBaseByIndex[take.index] ?? sourceTake; const v2vVideo = join(plan.workdir, V2V_DIRNAME, `${stem}.mp4`); const restoredTake = join(plan.workdir, RESTORED_DIRNAME, `${stem}.mp4`); if (confirmSpend) { // 1. Omni Flash V2V: base footage + composed prompt → annotated take. await opts.runner.runV2V({ index: take.index, baseFootage, prompt: take.prompt, outputPath: v2vVideo, }); // 2. Audio-restore: mux the ORIGINAL take audio onto the annotated video. await opts.runner.restoreAudio({ index: take.index, v2vVideo, originalTake: sourceTake, outputPath: restoredTake, }); } results.push({ index: take.index, anatomy: take.anatomy, baseFootage, audioSource: sourceTake, v2vVideo, restoredTake, status: confirmSpend ? 'ran' : 'planned', }); } // 3. Clip-stitch the audio-restored takes (in take order) into the final reel. const takeClips = results.map((r) => r.restoredTake); if (confirmSpend) { await opts.runner.stitch({ takeClips, outputPath: reelPath }); } const report: MotionOverlayExecuteReport = { schemaVersion: 1, mode, executed: confirmSpend, takes: results, reelPath, }; if (!confirmSpend) { report.hint = 'Dry run: no provider was called. Re-run with --execute --confirm-spend to render each take through Omni Flash V2V, restore the original audio, and stitch the reel.'; } return report; } // --------------------------------------------------------------------------- // Default (real) runner // --------------------------------------------------------------------------- /** * Build the audio-restore ffmpeg args (PURE). Takes the annotated VIDEO from the * V2V output and the ORIGINAL AUDIO from the source take, re-muxing them so the * reel keeps the untouched voiceover (CRITICAL RULE 1). * * -i -i * -map 0:v -map 1:a (video from input 0, audio from input 1) * -c:v copy (don't re-encode the annotated video) * -c:a aac (re-encode the restored audio to AAC for mp4 compat) * -shortest (cap to the shorter stream so a length mismatch can't * leave a trailing black/silent tail) * * * (`-y` is prepended by `runFfmpeg`, so it is NOT included here.) */ export function buildAudioRestoreArgs(v2vVideo: string, originalTake: string, outputPath: string): string[] { return [ '-i', v2vVideo, '-i', originalTake, '-map', '0:v', '-map', '1:a', '-c:v', 'copy', '-c:a', 'aac', '-shortest', outputPath, ]; } /** * The per-take omni-flash V2V transport. The CLI supplies the real one (a thin * wrapper over the native Flow V2V path: it submits the take's base footage as * the V2V edit reference with the composed prompt, polls to completion, and * writes the annotated take to `step.outputPath`). It is its OWN injectable seam * so the audio-restore + stitch halves of the default runner can be unit-tested * without a live provider. */ export type MotionOverlayV2VTransport = (step: MotionOverlayV2VStep) => Promise; export interface DefaultMotionOverlayRunnerOptions { /** The real V2V transport (CLI-supplied). Required — there is no offline default. */ v2v: MotionOverlayV2VTransport; /** Override the ffmpeg binary for the audio-restore mux. */ ffmpegBin?: string; } /** * Build the default {@link MotionOverlayStepRunner} that shells the REAL render * path: `v2v` for each take's omni-flash render, `runFfmpeg` for the audio-restore * mux ({@link buildAudioRestoreArgs}), and the assemble {@link stitch} clip-stitch * for the final reel. The V2V transport is injected so the audio + stitch wiring * stays independently testable; the whole runner is ONLY used under * `--execute --confirm-spend`, so tests never reach it (they inject a fake runner). */ export function createDefaultMotionOverlayRunner( opts: DefaultMotionOverlayRunnerOptions, ): MotionOverlayStepRunner { const ffmpegBin = opts.ffmpegBin; return { async runV2V(step) { await mkdir(dirname(step.outputPath), { recursive: true }); await opts.v2v(step); }, async restoreAudio(step) { await mkdir(dirname(step.outputPath), { recursive: true }); await runFfmpeg(buildAudioRestoreArgs(step.v2vVideo, step.originalTake, step.outputPath), { ...(ffmpegBin ? { ffmpegBin } : {}), }); }, async stitch(step) { await mkdir(dirname(step.outputPath), { recursive: true }); // Clip-stitch mode: stitch the audio-restored takes directly (each take is // an independent rendered clip, so demuxer/filter selection is automatic). await stitch( { segments: step.takeClips, outputPath: step.outputPath }, { ...(ffmpegBin ? { ffmpegBin } : {}) }, ); }, }; }