/** * music-video.ts — the vocal-synced, beat-exact music-video ASSEMBLER (the crown * jewel of the music-video lane). It turns a config (song + performer clips + * B-roll pools) into a finished cut whose performers land on their own vocals and * whose lips stay locked to the muxed song, with zero cumulative drift. * * This is a FULLY LOCAL assembler — no provider, no spend, only ffmpeg (already a * dependency). It composes the lane's pure pieces: * buildVocalMap (transcript → rap/hook/instrumental/outro) [or an explicit map] * → planVocalSync (performer-on-vocal, time-aligned, de-patterned B-roll) * → buildSegmentCutArgs (frame-exact `-frames:v` cut per segment) * → concat demuxer (-c copy) → ONE grade pass + mux the song. * * {@link buildMusicVideoPlan} is PURE (clip durations are injected via the config) * so the whole plan — every segment, every ffmpeg invocation — is unit-testable * offline. {@link runMusicVideo} is the thin execution shell: it writes the concat * list, spawns each step, then asserts the built master matches the planned * duration within one frame ({@link assertNoDrift}) so sync can never silently * regress. Dry-run returns the plan without spawning. */ import { promises as fs } from 'node:fs'; import * as path from 'node:path'; import { buildVocalMap, validateVocalMap, type WhisperSegment, } from './vocal-map.js'; import { planVocalSync, plannedDuration, type BrollPools, type PlannedSegment, type VocalSection, type VocalSectionType, } from './vocal-sync-plan.js'; import { buildSegmentCutArgs, assertNoDrift, CUT_DEFAULT_WIDTH, CUT_DEFAULT_HEIGHT, } from './assemble/cut-segment.js'; import { TARGET_FPS } from './assemble/animate-slides.js'; import { resolveGradeFilter, buildConcatListContent } from './assemble/stitch.js'; import { runFfmpeg, ffprobeDuration } from './assemble/ffmpeg.js'; /** A clip the assembler can cut from: an id plus its file and probed length. */ export interface ClipEntry { id: string; path: string; /** Source clip duration in seconds (so the planner stays pure). */ durationSec: number; } /** The music-video assembly config (an on-disk artifact or built in memory). */ export interface MusicVideoConfig { /** Path to the song audio that is muxed as the master's full audio track. */ song: string; /** Song length in seconds (the cut covers [0, songEnd]). */ songEnd: number; /** Output frame rate (default 24). */ fps?: number; /** Output width (default 1280). */ width?: number; /** Output height (default 720). */ height?: number; /** Grade id resolved via {@link resolveGradeFilter} (one pass over the whole cut). */ grade?: string; /** * Explicit vocal map (already carrying `performerClip`s). Wins over `transcript`. * Use this for a hand-authored map. */ vocalMap?: VocalSection[]; /** Whisper segments to auto-classify into a vocal map (when `vocalMap` is absent). */ transcript?: WhisperSegment[]; /** Override the auto hook detection (passed to {@link buildVocalMap}). */ hookKeywords?: string[]; /** * Performer clip id per vocal type, attached to the AUTO-built map * (rap → rapper clip, hook → singer clip). Ignored when `vocalMap` is explicit. */ performers?: { rap?: string; hook?: string }; /** B-roll clip ids grouped by role. */ pools: BrollPools; /** Clip registry: every id referenced by `pools`/`performers`/`vocalMap`. */ clips: ClipEntry[]; /** Beat timestamps (seconds); empty = cut on the section cadence only. */ beats?: number[]; /** Seconds-per-cut overrides by section type. */ cutLengths?: Partial>; /** Output master path (default `/master.mp4`). */ output?: string; } /** One ordered ffmpeg invocation in the assembly plan (args only, no spawn). */ export interface MusicVideoStep { kind: 'segment-cut' | 'concat-demuxer' | 'grade-mux'; /** Everything AFTER `ffmpeg -y` (so it matches {@link runFfmpeg}). */ args: string[]; /** Human label for logs/plan output. */ label: string; /** For the concat step: the concat-list file content to write before spawning. */ concatListContent?: string; /** For the concat step: where that list file is written. */ concatListPath?: string; } export interface MusicVideoPlan { schemaVersion: 1; vocalMap: VocalSection[]; segments: PlannedSegment[]; plannedDurationSec: number; width: number; height: number; fps: number; grade: string; steps: MusicVideoStep[]; segmentDir: string; concatVideo: string; output: string; } /** Attach the per-type performer clip to every vocal section of an auto-built map. */ function attachPerformers( map: VocalSection[], performers: { rap?: string; hook?: string } | undefined, ): VocalSection[] { if (!performers) return map; return map.map((s) => { const clip = s.type === 'rap' ? performers.rap : s.type === 'hook' ? performers.hook : undefined; return clip ? { ...s, performerClip: clip } : s; }); } /** * Build the full, frame-exact assembly plan. PURE — no fs, no spawn. `workDir` is * only used to compute step output paths; nothing is written here. Throws if a * referenced clip id is missing from the registry or the vocal map is invalid. */ export function buildMusicVideoPlan(config: MusicVideoConfig, workDir: string): MusicVideoPlan { const fps = config.fps ?? TARGET_FPS; const width = config.width ?? CUT_DEFAULT_WIDTH; const height = config.height ?? CUT_DEFAULT_HEIGHT; const grade = resolveGradeFilter(config.grade); // 1. resolve the vocal map (explicit wins; else auto-classify + attach performers). let vocalMap: VocalSection[]; if (config.vocalMap && config.vocalMap.length > 0) { vocalMap = config.vocalMap; } else { if (!config.transcript) { throw new Error('buildMusicVideoPlan: config needs either vocalMap or transcript.'); } const auto = buildVocalMap(config.transcript, { songEnd: config.songEnd, ...(config.hookKeywords ? { hookKeywords: config.hookKeywords } : {}), }); vocalMap = attachPerformers(auto, config.performers); } const issues = validateVocalMap(vocalMap, config.songEnd); if (issues.length > 0) { throw new Error(`buildMusicVideoPlan: invalid vocal map: ${issues.join('; ')}`); } // 2. clip registry → duration lookup + path lookup (pure planner inputs). const byId = new Map(config.clips.map((c) => [c.id, c])); const clipDuration = (id: string): number => { const c = byId.get(id); if (!c) throw new Error(`buildMusicVideoPlan: clip "${id}" referenced but not in the registry.`); return c.durationSec; }; const clipPath = (id: string): string => { const c = byId.get(id); if (!c) throw new Error(`buildMusicVideoPlan: clip "${id}" referenced but not in the registry.`); return c.path; }; // 3. plan the vocal-synced segment list. const segments = planVocalSync({ vocalMap, beats: config.beats ?? [], songEnd: config.songEnd, pools: config.pools, clipDuration, fps, ...(config.cutLengths ? { cutLengths: config.cutLengths } : {}), }); if (segments.length === 0) { throw new Error('buildMusicVideoPlan: planner produced no segments (check pools/vocal map).'); } // 4. compile the ffmpeg steps: frame-exact cut per segment → concat → grade + mux. const segmentDir = path.join(workDir, 'segments'); const concatVideo = path.join(workDir, 'concat-silent.mp4'); const concatListPath = path.join(workDir, 'concat-list.txt'); const output = config.output ?? path.join(workDir, 'master.mp4'); const steps: MusicVideoStep[] = []; const segmentPaths: string[] = []; segments.forEach((seg, i) => { const outputPath = path.join(segmentDir, `seg-${String(i).padStart(4, '0')}.mp4`); segmentPaths.push(outputPath); steps.push({ kind: 'segment-cut', label: `cut ${i} ${seg.performer ? 'performer' : 'b-roll'} ${seg.clip} @${seg.inSec}s ×${seg.durationSec}s`, args: buildSegmentCutArgs({ clipPath: clipPath(seg.clip), inSeconds: seg.inSec, durationSec: seg.durationSec, outputPath, width, height, fps, keepAudio: false, }), }); }); steps.push({ kind: 'concat-demuxer', label: `concat ${segmentPaths.length} segments (-c copy)`, concatListContent: buildConcatListContent(segmentPaths), concatListPath, args: ['-nostdin', '-f', 'concat', '-safe', '0', '-i', concatListPath, '-c', 'copy', concatVideo], }); // ONE grade pass + mux the song as the full audio track. Re-encode video only // when a grade is applied; otherwise copy the concatenated video stream. const gradeMuxArgs: string[] = ['-nostdin', '-i', concatVideo, '-i', config.song]; if (grade) { gradeMuxArgs.push('-vf', grade, '-c:v', 'libx264', '-crf', '18', '-preset', 'medium', '-pix_fmt', 'yuv420p'); } else { gradeMuxArgs.push('-c:v', 'copy'); } gradeMuxArgs.push( '-map', '0:v:0', '-map', '1:a:0', '-c:a', 'aac', '-b:a', '192k', '-shortest', output, ); steps.push({ kind: 'grade-mux', label: grade ? `grade (${config.grade}) + mux song` : 'mux song (no grade)', args: gradeMuxArgs, }); return { schemaVersion: 1, vocalMap, segments, plannedDurationSec: plannedDuration(segments), width, height, fps, grade, steps, segmentDir, concatVideo, output, }; } export interface RunMusicVideoOptions { /** Plan only — do not write files or spawn ffmpeg. */ dryRun?: boolean; /** Override the ffmpeg binary. */ ffmpegBin?: string; /** Override the ffprobe binary (drift check). */ ffprobeBin?: string; /** Skip the post-build drift assertion (e.g. when ffprobe is unavailable). */ skipDriftCheck?: boolean; } export interface RunMusicVideoResult { output: string; plannedDurationSec: number; builtDurationSec: number; segmentCount: number; performerCuts: number; brollCuts: number; grade: string; dryRun: boolean; } /** * Execute the assembly plan: mkdir the work dirs, run every step in order, write * the concat list before its step, then assert the built master matches the plan * within one frame. On `dryRun`, returns the planned metrics without touching disk. */ export async function runMusicVideo( plan: MusicVideoPlan, opts: RunMusicVideoOptions = {}, ): Promise { const performerCuts = plan.segments.filter((s) => s.performer).length; const base: Omit = { output: plan.output, plannedDurationSec: plan.plannedDurationSec, segmentCount: plan.segments.length, performerCuts, brollCuts: plan.segments.length - performerCuts, grade: plan.grade, }; if (opts.dryRun) { return { ...base, builtDurationSec: plan.plannedDurationSec, dryRun: true }; } await fs.mkdir(plan.segmentDir, { recursive: true }); await fs.mkdir(path.dirname(plan.output), { recursive: true }); for (const step of plan.steps) { if (step.concatListContent && step.concatListPath) { await fs.writeFile(step.concatListPath, step.concatListContent, 'utf8'); } await runFfmpeg(step.args, { ...(opts.ffmpegBin ? { ffmpegBin: opts.ffmpegBin } : {}) }); } let builtDurationSec = plan.plannedDurationSec; if (!opts.skipDriftCheck) { const ms = await ffprobeDuration(plan.output, { ...(opts.ffprobeBin ? { ffprobeBin: opts.ffprobeBin } : {}) }); builtDurationSec = ms / 1000; assertNoDrift(plan.plannedDurationSec, builtDurationSec, plan.fps); } return { ...base, builtDurationSec, dryRun: false }; }