/** * Lipsync closure protocol — the bilabial seal map (Joey 3.0 Cinema Director). * * Lipsync reads as fake for a diagnosable reason: the lyric was handed to the * model as an abstract instruction ("she sings the line") instead of as a score * of mouth mechanics. What the eye actually reads as real sync is the **lip * seal** — the moment both lips press fully together on a bilabial consonant. * Miss those and the mouth flaps plausibly but syncs to nothing. * * So this module turns a line of lyric or dialogue into a COUNTED, positioned * map of seals, which the prompt then states as a directive block. A count the * model can check itself against beats a paragraph of adjectives. * * Pure and deterministic — no I/O. */ /** * `hard` — a full bilabial seal: both lips press flat together and part again. * B, P (plosives) and M (nasal) all require complete closure, so all three * count. These are what the eye reads. * * `soft` — labiodental: the upper teeth touch the lower lip (F, V). Visible and * worth describing, but NOT a lip seal, so it is deliberately excluded from the * count. Counting it inflates the number and the model starts inventing * closures that the audio does not contain. */ export type ClosureKind = 'hard' | 'soft'; export type ClosurePosition = 'initial' | 'medial' | 'final'; export interface LipsyncClosure { /** The letter producing the closure, uppercased. */ letter: string; kind: ClosureKind; /** The word it falls in, uppercased — the prompt names the word, not the index. */ word: string; /** Where in the word the closure lands; drives the "M ending X" phrasing. */ position: ClosurePosition; } /** * Words whose final `b` is silent, so the `mb` pair is ONE seal (the M), not * two. A naive letter scan reports two and the closure count comes out wrong, * which is worse than no count at all — the model is being told to produce a * seal the audio never contains. */ const SILENT_FINAL_MB = /mb$/i; /** * Word-initial silent `p`: `ps-` (psalm, psychic), `pn-` (pneumatic), * `pt-` (pterodactyl). The lips never close. */ const SILENT_INITIAL_P = /^p[snt]/i; /** * `ph` is an F sound — labiodental, not bilabial. Treated as a soft closure * wherever it appears, and the `p` in it never counts as a seal. */ const PH_DIGRAPH = /ph/gi; /** * Where the word effectively ENDS for closure-position purposes. * * English magic-e ("time", "come", "name") puts a silent letter after the * consonant that actually closes the word, so an orthographic index reports * that seal as medial. The protocol's whole phrasing turns on this — "the M * ending TIME" tells the model the lips seal and the word is over, where "the M * inside TIME" implies something follows it. Trim a single silent final `e`, * but only after a consonant: "free" and "see" keep theirs, and two-letter * "be"/"me" are pronounced. */ function effectiveLength(word: string): number { if (word.length > 3 && /[^aeiou]e$/i.test(word)) return word.length - 1; return word.length; } function classifyPosition(index: number, length: number): ClosurePosition { if (index === 0) return 'initial'; if (index >= length - 1) return 'final'; return 'medial'; } /** * Scan one line into its ordered closure map. * * Known limits, stated rather than silently wrong: this is orthographic, not * phonetic. It handles the three traps that actually change the count in * practice (silent final `mb`, silent initial `p`, and `ph` as an F sound) and * does not attempt full grapheme-to-phoneme conversion — a homograph like * "read" is not disambiguated, though neither pronunciation changes a seal. * Verify the count against a hand-count before a paid render. */ export function buildClosureMap(line: string): LipsyncClosure[] { const closures: LipsyncClosure[] = []; const words = line.split(/\s+/).filter(Boolean); for (const raw of words) { // Strip surrounding punctuation but keep intra-word marks (don't → DON'T). const word = raw.replace(/^[^\p{L}']+|[^\p{L}']+$/gu, ''); if (!word) continue; const upper = word.toUpperCase(); // Mark the positions consumed by `ph` so the p is not also counted as a seal. const phPositions = new Set(); for (const match of word.matchAll(PH_DIGRAPH)) { if (match.index !== undefined) phPositions.add(match.index); } const silentFinalB = SILENT_FINAL_MB.test(word); const silentInitialP = SILENT_INITIAL_P.test(word); const effLength = effectiveLength(word); for (let i = 0; i < word.length; i += 1) { const letter = word[i]!.toUpperCase(); if (phPositions.has(i)) { // The `p` of `ph` — an F sound, so a teeth-on-lip contact, not a seal. closures.push({ letter: 'PH', kind: 'soft', word: upper, position: classifyPosition(i, effLength) }); continue; } // The `h` of `ph` was already reported by the `p` above. if (letter === 'H' && phPositions.has(i - 1)) continue; if (letter === 'B' || letter === 'M' || letter === 'P') { if (letter === 'B' && silentFinalB && i === word.length - 1) continue; if (letter === 'P' && silentInitialP && i === 0) continue; closures.push({ letter, kind: 'hard', word: upper, position: classifyPosition(i, effLength) }); continue; } if (letter === 'F' || letter === 'V') { closures.push({ letter, kind: 'soft', word: upper, position: classifyPosition(i, effLength) }); } } } return closures; } /** The hard seals only — the number the prompt actually states. */ export function hardClosures(map: LipsyncClosure[]): LipsyncClosure[] { return map.filter((c) => c.kind === 'hard'); } const POSITION_WORD: Record = { initial: 'opening', medial: 'inside', final: 'ending', }; function describeClosure(c: LipsyncClosure): string { return `the ${c.letter} ${POSITION_WORD[c.position]} "${c.word}"`; } /** * THE PATTERN OF CLOSURES — the counted seal directive. * * The count is the load-bearing part. "Sing the line clearly" is unfalsifiable; * "there are four complete lip seals and here is where each one lands" is * something the model can hold itself to. */ export function closureCountBlock(map: LipsyncClosure[]): string { const hard = hardClosures(map); if (hard.length === 0) { return ( 'THE PATTERN OF CLOSURES: this line contains no bilabial consonants, so the lips never fully seal. ' + 'The mouth stays actively shaped through every vowel and never drifts to a slack half-open rest position.' ); } const soft = map.filter((c) => c.kind === 'soft'); const softClause = soft.length > 0 ? ` The teeth touch the lower lip on ${soft.map(describeClosure).join(' and ')} — visible contact, but not a lip seal and not counted among the ${hard.length}.` : ''; return ( `THE PATTERN OF CLOSURES: ${hard.length} complete lip seal${hard.length === 1 ? '' : 's'} across this line — ` + `${hard.map(describeClosure).join(', ')}. On every one of them both lips press fully flat together, ` + 'seal shut, hold for a beat, and part again — a complete and unmistakable closure, not a narrowing. ' + `The mouth is never lazily half-open and never mumbling between them.${softClause}` ); } /** * The singing-is-the-subject directive. Promoted high because lipsync competes * with every other directive block for attention, and loses by default. */ export function singingPrimaryBlock(subjectHandle: string, seconds: number): string { // Subject handles are visual descriptors ("the woman with the long jet-black // hair"), so they arrive lowercase and land mid-paragraph at a sentence start. const handle = subjectHandle.charAt(0).toUpperCase() + subjectHandle.slice(1); return ( `THE SINGING IS THE PRIMARY SUBJECT OF THIS SEQUENCE — every other element is secondary to it. ` + `${handle} sings out loud, full voice, mouth open and working hard, for all ${seconds} seconds without stopping. ` + 'This is a vocal delivered straight down the lens, not a performer mouthing along to playback. ' + 'The mouth is the focus of every shot.' ); } /** * Mouth-visibility lock. Pairs with the closure map: a perfectly scored seal * that happens behind a hand or a hair fall reads as no seal at all. */ export function mouthVisibilityBlock(): string { return ( 'THE MOUTH IS ALWAYS VISIBLE AND ALWAYS READABLE: the face stays turned toward the lens and the mouth is ' + 'unobstructed, frontal, and clearly readable in every single frame of every shot — through the camera movement, ' + 'through any cant, and through every change in the light. Nothing ever covers it: no hand, no hair, no arm, ' + 'no prop, and no other body. The mouth never turns away from the lens.' ); } /** * Sole-audio-source lock for a submitted vocal reference. * * The reference SHAPE matters as much as the lock: a bare MP3/WAV drifts to a * generic accent, while the same audio as the track of a black-frame video * holds the voice (this is what `vclaw video voice-clone` builds). Passing no * vocal reference at all means the take can never sync, however well the * closures are scored — so this block assumes one is attached and says so. */ export function soleAudioSourceBlock(referenceSlot: string): string { return ( `SOUND BED: ${referenceSlot} is the sole and complete audio source for this sequence. Generate no additional ` + 'audio of any kind — no room tone, no foley, no ambience, no breath, no added dialogue, and no music. ' + `That clip also owns all internal timing: do not impose any invented rhythm on the singing, and sync every ` + 'syllable, vowel opening, and lip seal precisely to the vocal in it.' ); } export interface LipsyncBlocksInput { /** Visual descriptor of the performer — never a proper name. */ subjectHandle: string; /** The sung or spoken line, verbatim. */ line: string; durationSeconds: number; /** Reference slot carrying the vocal, e.g. `@video1`. Omit for no audio lock. */ audioReferenceSlot?: string; } /** * The full protocol as an ordered block list, ready to prepend to a packet. * Ordering is the protocol: singing first (it out-competes everything else), * then the counted seals, then the visibility lock that makes them readable. */ export function lipsyncBlocks(input: LipsyncBlocksInput): string[] { const map = buildClosureMap(input.line); const blocks = [ singingPrimaryBlock(input.subjectHandle, input.durationSeconds), `THE LINE, VERBATIM: "${input.line.trim()}"`, closureCountBlock(map), mouthVisibilityBlock(), ]; if (input.audioReferenceSlot) { blocks.push(soleAudioSourceBlock(input.audioReferenceSlot)); } return blocks; }