/** * Motion-overlay plan orchestrator (Phase 2). * * `planMotionOverlay` is the dry/plan front door wired into `vclaw video * motion-overlay`. It runs the full plan pipeline: * ingest.probeVideo → extractAudio → transcribe (or --transcript) → slice → * ingest.cutTake per take → extractFrames → readFrameObservations → * analyze-reel → compose-prompt per take → plan → write work folder. * * Every side effect (ffmpeg, ffprobe, Gemini STT) is injected via `deps`, so the * whole orchestrator runs offline in tests with fakes. NO provider spend happens * here — this is the plan/dry path; execution (V2V) lands in Phase 4. */ import { join } from 'node:path'; import { copyFile, mkdir, writeFile } from 'node:fs/promises'; import { analyzeTake } from './analyze-reel.js'; import { composePrompt } from './compose-prompt.js'; import { cutTake, extractAudio, extractFrames, probeVideo, readFrameObservations, type IngestDeps, } from './ingest.js'; import { buildMotionOverlayPlan, takeStem } from './plan.js'; import { resolveStyle } from './motion-style.js'; import { planSplits } from './slice.js'; import { loadTranscript, transcribeAudio, type TranscribeDeps } from './transcribe.js'; import { writeMotionOverlayReview } from './preview.js'; import { writeWorkFolder } from './write.js'; import type { MotionLayout, MotionOverlayPlan, MotionStyleId, Transcript } from './types.js'; /** Human-readable language label for the on-screen-text GLOBAL RULE. */ const LANGUAGE_NAMES: Record = { en: 'English', pt: 'Portuguese', es: 'Spanish', fr: 'French', de: 'German', it: 'Italian', hi: 'Hindi', ja: 'Japanese', zh: 'Chinese', }; /** Map a BCP-47 code (or "auto") to a human language label for the prompt. */ export function languageLabel(code: string): string { const key = code.trim().toLowerCase(); if (!key || key === 'auto') return 'the spoken language'; return LANGUAGE_NAMES[key] ?? code.trim(); } export interface PlanMotionOverlayArgs { /** Absolute path to the input video. */ input: string; /** Absolute work-folder path to emit into. */ workdir: string; layout: MotionLayout; style: MotionStyleId; /** Requested STT language ("auto" lets Gemini detect). */ lang: string; /** Omni hard per-clip limit (seconds). */ maxTakeSeconds: number; /** Optional accent override (hex or name). */ accent?: string; /** Optional brand-derived accent (wins over --accent). */ brandAccent?: string; /** Optional bring-your-own transcript path (skips STT). */ transcriptPath?: string; /** Number of reference frames to sample (default 3). */ frameCount?: number; /** Emit the `review/review.html` approval surface alongside the plan. */ preview?: boolean; /** Deterministic timestamp for the manifest (tests). */ generatedAt?: string; } export interface PlanMotionOverlayDeps extends IngestDeps, TranscribeDeps {} export interface PlanMotionOverlayResult { plan: MotionOverlayPlan; manifestPath: string; readmePath: string; promptPaths: string[]; /** Path to `review/review.html` when `preview` was requested, else undefined. */ reviewPath?: string; } /** * Run the full plan pipeline and emit the work folder. Returns the assembled * plan + the paths written. Pure orchestration over injected side effects. */ export async function planMotionOverlay( args: PlanMotionOverlayArgs, deps: PlanMotionOverlayDeps = {}, ): Promise { const ingestDeps: IngestDeps = { ...(deps.runFfmpeg ? { runFfmpeg: deps.runFfmpeg } : {}), ...(deps.probe ? { probe: deps.probe } : {}), }; const transcribeDeps: TranscribeDeps = { ...(deps.transcriber ? { transcriber: deps.transcriber } : {}), ...(deps.fetcher ? { fetcher: deps.fetcher } : {}), ...(deps.endpoint ? { endpoint: deps.endpoint } : {}), }; // 0. Create the work-folder subdirs up front so ingest can write into them // (the README/manifest are emitted by writeWorkFolder at the end). await Promise.all([ mkdir(join(args.workdir, 'source'), { recursive: true }), mkdir(join(args.workdir, 'takes'), { recursive: true }), mkdir(join(args.workdir, 'frames'), { recursive: true }), mkdir(join(args.workdir, 'prompts'), { recursive: true }), ]); // 1. Probe the input. const probe = await probeVideo(args.input, ingestDeps); // 2. Extract audio into the work folder's source/ dir, then transcribe. const audioPath = join(args.workdir, 'source', 'audio.aac'); await extractAudio(args.input, audioPath, ingestDeps); let transcript: Transcript; if (args.transcriptPath) { transcript = await loadTranscript(args.transcriptPath); } else { transcript = await transcribeAudio(audioPath, args.lang, transcribeDeps); } // Persist the transcript alongside source/ for the operator + execute phase. const transcriptOut = join(args.workdir, 'source', 'transcript.json'); await writeFile(transcriptOut, `${JSON.stringify(transcript, null, 2)}\n`, 'utf-8'); // 3. Slice into takes at sentence boundaries ≤ max. const splits = planSplits(transcript, args.maxTakeSeconds); // 4. Cut each take frame-accurately + sample reference frames. for (const split of splits) { const clipPath = join(args.workdir, 'takes', `${takeStem(split)}.mp4`); await cutTake(args.input, split.start, split.end, clipPath, ingestDeps); } const frameFiles = await extractFrames( args.input, join(args.workdir, 'frames'), probe.durationSeconds, args.frameCount ?? 3, ingestDeps, ); const frames = readFrameObservations(probe, frameFiles); // 5. Resolve style + analyze + compose per take. const resolvedStyle = resolveStyle(args.style, { ...(args.accent ? { accent: args.accent } : {}), ...(args.brandAccent ? { brand: { accent: args.brandAccent } } : {}), }); const onScreenLanguage = languageLabel(transcript.language || args.lang); const analyses = splits.map((split) => analyzeTake(split, transcript.segments, splits.length)); const prompts = splits.map((split, i) => composePrompt({ index: split.index, split, segments: transcript.segments, layout: args.layout, style: resolvedStyle, analysis: analyses[i], frames, language: onScreenLanguage, durationSeconds: split.end - split.start, }), ); // 6. Assemble the manifest. const plan = buildMotionOverlayPlan({ input: { path: args.input, durationSeconds: probe.durationSeconds, width: probe.width, height: probe.height, aspect: probe.aspect, }, layout: args.layout, style: args.style, accent: resolvedStyle.accent, language: transcript.language || args.lang, workdir: args.workdir, splits, analyses, prompts, }); // 7. Write the work folder (README + manifest + per-take prompts). const written = await writeWorkFolder(args.workdir, plan, { ...(args.generatedAt ? { generatedAt: args.generatedAt } : {}), }); // Copy the original into source/ for provenance (best-effort; non-fatal). await copyFile(args.input, join(args.workdir, 'source', 'original.mp4')).catch(() => {}); const stampedPlan: MotionOverlayPlan = { ...plan, generatedAt: args.generatedAt ?? plan.generatedAt }; // Optionally emit the review/review.html approval surface (the contract). let reviewPath: string | undefined; if (args.preview) { const review = await writeMotionOverlayReview(args.workdir, stampedPlan); reviewPath = review.reviewPath; } return { plan: stampedPlan, manifestPath: written.manifestPath, readmePath: written.readmePath, promptPaths: written.promptPaths, ...(reviewPath ? { reviewPath } : {}), }; }