/** * vocal-sync-plan.ts — the planner that makes a music-video edit feel "in sync": * it pins each PERFORMER to the moment their own vocal is heard, and keeps the * baked lip-sync locked to the muxed song. * * Given a VOCAL MAP (the song segmented by timestamp into rap / hook / * instrumental / outro, each vocal section carrying its lip-synced performer * clip) plus the beat grid and B-roll pools, it emits a frame-aligned segment * list (`clip | inSec | durationSec`) for {@link buildSegmentCutArgs}. PURE and * deterministic given an injected `rng` and `clipDuration` — no I/O, no ffmpeg. * * The three lessons it encodes (each learned by getting it wrong first): * 1. PERFORMER-ON-VOCAL: a music video needs the rapper on the rap and the * singer on the hook — assigning performers to GUESSED time blocks fights the * audio. Sections come from the transcribed vocal map, not a shot script. * 2. TIME-ALIGNED SCRUB: a performer cut shows its clip at in-point * `(songTime - sectionStart)`, so the baked lips stay locked to the song even * across B-roll cutaways (cut away and back → correct lip position). The * performer holds ~2 of every 3 cuts and OPENS each vocal section. * 3. FRAME-EXACT CONTINUOUS TIMELINE: walk one timeline from t=0 (no per-section * reset — that drops the intro and offsets every cue) and snap every cut to * the frame grid so the concatenated video-time == song-time. Paired with * `-frames:v` cutting, total drift is < 1 frame. * Plus DE-PATTERNING: B-roll draws from a shuffle-bag (never round-robin) with a * stepped in-point on reuse, so recurring footage doesn't read as a loop. */ export type VocalSectionType = 'rap' | 'hook' | 'instrumental' | 'outro'; export interface VocalSection { /** Section start in song seconds. */ start: number; /** Section end in song seconds. */ end: number; type: VocalSectionType; /** * The lip-synced performer clip id for this vocal section (rap → rapper clip, * hook → singer clip). Omitted for instrumental/outro (B-roll only). */ performerClip?: string; } /** B-roll clip ids grouped by role; performers come from the vocal map, not here. */ export interface BrollPools { action: string[]; atmo: string[]; trio: string[]; vanish: string[]; } export interface PlanVocalSyncInput { /** Contiguous, ordered vocal map covering [0, songEnd]. */ vocalMap: VocalSection[]; /** Beat timestamps (seconds), e.g. from aubiotrack. */ beats: number[]; /** Song length in seconds. */ songEnd: number; pools: BrollPools; /** Duration (seconds) of a clip id — injected so the planner stays pure. */ clipDuration: (clipId: string) => number; /** Output frame rate (default 24). */ fps?: number; /** Seconds-per-cut by section type. Verses cut fast; hooks hold longer. */ cutLengths?: Partial>; /** Injected RNG (default Math.random) — seed it for deterministic tests. */ rng?: () => number; } export interface PlannedSegment { clip: string; /** Frame-aligned input-seek point. */ inSec: number; /** Frame-aligned duration. */ durationSec: number; /** True when this cut is a lip-synced performer (vs B-roll). */ performer: boolean; } const DEFAULT_CUTS: Record = { rap: 1.3, hook: 2.4, instrumental: 2.0, outro: 3.1, }; const BROLL_ORDER: Record> = { rap: ['action', 'trio'], hook: ['atmo', 'trio'], instrumental: ['atmo', 'trio', 'action'], outro: ['vanish', 'atmo'], }; function shuffle(arr: T[], rng: () => number): T[] { const a = [...arr]; for (let i = a.length - 1; i > 0; i--) { const j = Math.floor(rng() * (i + 1)); [a[i], a[j]] = [a[j], a[i]]; } return a; } export function planVocalSync(input: PlanVocalSyncInput): PlannedSegment[] { const fps = input.fps ?? 24; const rng = input.rng ?? Math.random; const cuts = { ...DEFAULT_CUTS, ...input.cutLengths }; const beats = input.beats; const nearestBeat = (t: number): number => beats.length === 0 ? t : beats.reduce((p, b) => (Math.abs(b - t) < Math.abs(p - t) ? b : p), beats[0]); // Boundaries and section lookup are FRAME-based, using the same rounding the // walk uses. The previous float version (`find(b => b > t + 0.05)`) skipped a // boundary whenever frame rounding put the walk within 0.05s before it, so // the previous section's type/performer bled a full cut into the next // section for roughly half of all boundaries. const songEndF = Math.round(input.songEnd * fps); const boundaryFrames = input.vocalMap.map((s) => Math.round(s.start * fps)).concat(songEndF); const sectionAtFrame = (frame: number): VocalSection => { for (const s of input.vocalMap) { if (frame >= Math.round(s.start * fps) && frame < Math.round(s.end * fps)) return s; } return { start: 0, end: input.songEnd, type: 'instrumental' }; }; const nextBoundaryFrame = (frame: number): number => boundaryFrames.find((bf) => bf > frame) ?? songEndF; const segs: PlannedSegment[] = []; const useCount: Record = {}; let lastClip: string | null = null; const bags: Partial> = {}; const draw = (poolKey: keyof BrollPools): string | null => { const pool = input.pools[poolKey]; if (!pool || pool.length === 0) return null; let bag = bags[poolKey]; if (!bag || bag.length === 0) bag = bags[poolKey] = shuffle(pool, rng); let clip = bag.shift() as string; if (clip === lastClip && bag.length > 0) { bag.push(clip); clip = bag.shift() as string; } return clip; }; const emitBroll = (clip: string, durationSec: number): void => { const od = input.clipDuration(clip); // Fail at PLAN time when the source can't supply the cut (plus the 0.05s // min in-point and 0.15s tail margin the in-point stepper reserves). // Previously the Math.max(0.05) floor masked this and it surfaced // post-render as a misleading drift failure. if (od < durationSec + 0.2) { throw new Error( `planVocalSync: B-roll clip "${clip}" is ${od.toFixed(2)}s but a ${durationSec.toFixed(2)}s cut (+0.2s margin) is planned — use longer pool footage or shorter cutLengths.`, ); } lastClip = clip; const n = (useCount[clip] = (useCount[clip] ?? 0) + 1); const room = Math.max(0.05, od - durationSec - 0.15); const inSec = +(((n - 1) * (durationSec + 0.5) + 0.2) % room).toFixed(4); segs.push({ clip, inSec, durationSec, performer: false }); }; const emitPerformer = (clip: string, wantIn: number, durationSec: number): void => { const od = input.clipDuration(clip); // An undersized lip-synced take cannot cover its section's cuts; the old // clamp silently scrubbed to 0 and desynced the lips post-render. if (od < durationSec) { throw new Error( `planVocalSync: performer clip "${clip}" is ${od.toFixed(2)}s but a ${durationSec.toFixed(2)}s cut is planned — the lip-synced take must cover its section's cuts.`, ); } lastClip = clip; const inSec = Math.max(0, Math.min(wantIn, od - durationSec)); segs.push({ clip, inSec: +inSec.toFixed(4), durationSec, performer: true }); }; // Walk an INTEGER-FRAME timeline so cumulative video-time == song-time exactly // (float `t` accumulates rounding and would drift cuts off section boundaries). const minFrames = Math.round(0.3 * fps); // skip sub-0.3s slivers let tf = 0; let lastType: VocalSectionType | null = null; let cutIdx = 0; let brollRot = 0; while (tf < songEndF - Math.round(0.2 * fps)) { const t = tf / fps; const sec = sectionAtFrame(tf); const spc = cuts[sec.type]; if (sec.type !== lastType) { cutIdx = 0; brollRot = 0; lastType = sec.type; } let cutSec = nearestBeat(t + spc); if (cutSec <= t + 0.25) cutSec = t + spc; let cf = Math.round(cutSec * fps); const nbf = nextBoundaryFrame(tf); if (cf > nbf) cf = nbf; if (cf > songEndF) cf = songEndF; if (cf <= tf) { tf += 1; // guarantee forward progress at a degenerate boundary continue; } const durFrames = cf - tf; if (durFrames < minFrames) { tf = cf; continue; } const durationSec = durFrames / fps; if (sec.performerClip && cutIdx % 3 !== 2) { emitPerformer(sec.performerClip, (tf - Math.round(sec.start * fps)) / fps, durationSec); } else { const order = BROLL_ORDER[sec.type]; const key = order[brollRot++ % order.length]; const clip = draw(key) ?? draw('trio') ?? draw('atmo'); if (!clip) { tf = cf; continue; } emitBroll(clip, durationSec); } tf = cf; cutIdx += 1; } return segs; } /** Total planned duration (seconds) — feed to {@link assertNoDrift} against the built file. */ export function plannedDuration(segs: PlannedSegment[]): number { return +segs.reduce((a, s) => a + s.durationSec, 0).toFixed(4); }