/** * vocal-map.ts — turn a transcript into the VOCAL MAP the music-video planner * needs: the song segmented by timestamp into rap / hook / instrumental / outro. * * {@link buildVocalMap} is a PURE classifier over whisper-style segments * (`{start, end, text}`). The hook is the SUSTAINED, sung passage; the verses are * the dense, fast-delivered rap. The discriminating signal is WORD DENSITY * (words per second): a sung chorus stretches few words over many seconds (low * wps) while rap packs many words into little time (high wps). The classifier * finds the natural split by the largest gap in the sorted densities. ("Repeated * line" is deliberately NOT the signal — verses repeat too, which mislabels them; * density does not have that failure.) `hookKeywords` overrides when the caller * knows the chorus words. Everything else with words is RAP; gaps with no words * are INSTRUMENTAL; the trailing gap is the OUTRO. The result is a contiguous, * gap-free section list covering [0, songEnd] — exactly the shape the planner * consumes (the per-section performer clip is attached later by the assembler, * rap → rapper clip / hook → singer clip). * * The map can also be authored by hand (an explicit artifact) and fed straight to * the planner; {@link validateVocalMap} guards either source. */ import type { VocalSection, VocalSectionType } from './vocal-sync-plan.js'; export interface WhisperSegment { start: number; end: number; text: string; } export interface BuildVocalMapOptions { /** Song length in seconds (the map covers [0, songEnd]). */ songEnd: number; /** * Phrases that mark the HOOK. If given, a segment is hook when its text * contains any keyword. If omitted, hooks are auto-detected by word density. */ hookKeywords?: string[]; /** A silent stretch ≥ this is its own INSTRUMENTAL section (default 2.0s). */ minInstrumentalSec?: number; /** Vocal spans of the same type within this gap are merged (default 1.2s). */ mergeGapSec?: number; /** * Minimum words-per-second gap to declare a hook/rap split (default 0.4). If * the song's densities don't separate this much (e.g. all-rap or all-sung), * everything is treated as RAP — pass hookKeywords to force a hook. */ hookDensityGapMin?: number; } const normalize = (text: string): string => text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, '').replace(/\s+/g, ' ').trim(); /** Classify whisper segments into a contiguous vocal map. PURE. */ export function buildVocalMap(segments: WhisperSegment[], options: BuildVocalMapOptions): VocalSection[] { const { songEnd } = options; const minGap = options.minInstrumentalSec ?? 2.0; const mergeGap = options.mergeGapSec ?? 1.2; // 1. classify each spoken segment as rap (dense) or hook (sustained), by the // natural gap in word density. wps = words / duration. const spoken = segments.filter((s) => normalize(s.text).length > 0 && s.end > s.start); const keywords = options.hookKeywords?.map(normalize).filter(Boolean); const wpsOf = (s: WhisperSegment): number => normalize(s.text).split(' ').filter(Boolean).length / Math.max(0.5, s.end - s.start); const densities = spoken.map(wpsOf).sort((a, b) => a - b); let splitGap = 0; let threshold = -1; for (let i = 1; i < densities.length; i++) { const g = densities[i] - densities[i - 1]; if (g > splitGap) { splitGap = g; threshold = (densities[i] + densities[i - 1]) / 2; } } const minGapWps = options.hookDensityGapMin ?? 0.4; const isHook = (s: WhisperSegment): boolean => { if (keywords && keywords.length > 0) return keywords.some((k) => normalize(s.text).includes(k)); // below the density split = the sustained, sung cluster = hook. Without a // meaningful separation, default to RAP (caller can pass hookKeywords). return splitGap >= minGapWps && wpsOf(s) < threshold; }; type Span = { start: number; end: number; type: VocalSectionType }; const spans: Span[] = spoken .map((s) => ({ start: s.start, end: s.end, type: (isHook(s) ? 'hook' : 'rap') as VocalSectionType })) .sort((a, b) => a.start - b.start); // 2. merge consecutive same-type spans within mergeGap const merged: Span[] = []; for (const span of spans) { const last = merged[merged.length - 1]; if (last && last.type === span.type && span.start - last.end <= mergeGap) { last.end = Math.max(last.end, span.end); } else { merged.push({ ...span }); } } // 3. fill the timeline: large gaps -> instrumental, small gaps absorbed into the // following vocal section; the trailing gap -> outro. const sections: VocalSection[] = []; let cursor = 0; for (const span of merged) { // Transcripts can run past the declared song end (whisper timestamps beyond // the trimmed song). Spans are sorted, so everything from here on is out of // bounds — emitting them produced overlapping, beyond-songEnd sections. if (span.start >= songEnd) break; const gap = span.start - cursor; let start = cursor; if (gap >= minGap) { sections.push({ start: cursor, end: span.start, type: 'instrumental' }); start = span.start; cursor = span.start; } const end = Math.min(songEnd, span.end); if (end > start) { sections.push({ start, end, type: span.type }); cursor = end; } } const tail = songEnd - cursor; if (tail > 0.1) { sections.push({ start: cursor, end: songEnd, type: cursor > 0 ? 'outro' : 'instrumental' }); } else if (tail > 1e-6) { // A micro trailing gap (≤0.1s) is absorbed into the last section — leaving // it uncovered fails validateVocalMap's 1e-6 coverage check and hard-threw // the whole auto path on common transcripts. if (sections.length > 0) sections[sections.length - 1].end = songEnd; else sections.push({ start: 0, end: songEnd, type: 'instrumental' }); } return sections; } /** * Validate a vocal map (auto-built or hand-authored): contiguous, in-bounds, * gap-free coverage of [0, songEnd]. Returns a list of human-readable issues * (empty = valid). */ export function validateVocalMap(sections: VocalSection[], songEnd: number): string[] { const issues: string[] = []; if (sections.length === 0) { return ['vocal map is empty']; } if (Math.abs(sections[0].start) > 1e-6) issues.push(`first section starts at ${sections[0].start}, expected 0`); for (let i = 0; i < sections.length; i++) { const s = sections[i]; if (s.end <= s.start) issues.push(`section ${i} (${s.type}) has end <= start`); if (i > 0 && Math.abs(s.start - sections[i - 1].end) > 1e-6) { issues.push(`gap/overlap between section ${i - 1} and ${i} (${sections[i - 1].end} -> ${s.start})`); } } const last = sections[sections.length - 1]; if (Math.abs(last.end - songEnd) > 1e-6) issues.push(`last section ends at ${last.end}, expected songEnd ${songEnd}`); return issues; }