/** * lipsync.ts — audio-driven lip-sync: a still (or a character keyframe) + a vocal * track → a lip-synced, expressive talking-head clip, via the apiz/xskill * OmniHuman v1.5 model (`fal-ai/bytedance/omnihuman/v1.5`). * * This is distinct from motion-overlay's avatar-host (which makes the model speak * in its OWN generated voice): lipsync drives an EXTERNAL vocal — a rapper's verse, * a singer's hook, a narrator — so it is the building block the music-video lane * uses to put a performer's real vocal on their face. * * Flow: upload the image + audio (uguu) → submit OmniHuman → await → download → * NORMALIZE. The normalize step is load-bearing: OmniHuman returns 25fps and * sometimes odd dimensions, which break ffmpeg's frame-accurate seek downstream * (the assembler's `-ss`/`-frames:v` cutting then lands on the wrong frame). We * re-encode to CFR 24fps + even dimensions so the clip slots cleanly into the cut. * * Model limits (from the schema): 1080p caps audio at 30s, 720p at 60s — enforced * up front so a too-long vocal fails fast with an actionable message rather than a * provider error. Pure helpers ({@link buildOmniHumanParams}, * {@link buildLipsyncNormalizeArgs}) are unit-tested; {@link runLipsync} is the * thin executor with injectable transports. `--dry-run` plans without spending. */ import { hostPublicUrl } from './providers/public-host.js'; import { xskillSubmit, xskillAwait, xskillDownload, type XskillAwaitOptions } from './providers/xskill.js'; import { probeMedia } from './final-media.js'; import { runFfmpeg } from './assemble/ffmpeg.js'; import { copyFile, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; /** The apiz/xskill model id for OmniHuman v1.5 audio-driven video. */ export const OMNIHUMAN_MODEL_ID = 'fal-ai/bytedance/omnihuman/v1.5'; export type LipsyncResolution = '720p' | '1080p'; /** Max input-audio seconds the model accepts at each resolution. */ export const OMNIHUMAN_AUDIO_LIMITS: Record = { '720p': 60, '1080p': 30 }; export interface OmniHumanParams { image_url: string; audio_url: string; resolution: LipsyncResolution; prompt?: string; turbo_mode?: boolean; } export interface BuildOmniHumanParamsInput { imageUrl: string; audioUrl: string; resolution?: LipsyncResolution; prompt?: string; turbo?: boolean; } /** Build the OmniHuman request params (PURE). Defaults to 1080p. */ export function buildOmniHumanParams(input: BuildOmniHumanParamsInput): OmniHumanParams { const params: OmniHumanParams = { image_url: input.imageUrl, audio_url: input.audioUrl, resolution: input.resolution ?? '1080p', }; if (input.prompt) params.prompt = input.prompt; if (input.turbo) params.turbo_mode = true; return params; } /** * Build the ffmpeg args (PURE) that NORMALIZE an OmniHuman clip to CFR `fps` * (default 24) + even dimensions, so frame-accurate seeking works downstream. * Keeps the (already lip-synced) audio. Everything AFTER `ffmpeg -y`. */ export function buildLipsyncNormalizeArgs(input: string, output: string, fps = 24): string[] { return [ '-nostdin', '-i', input, '-vf', `fps=${fps},scale=trunc(iw/2)*2:trunc(ih/2)*2,setsar=1`, '-c:v', 'libx264', '-crf', '18', '-preset', 'medium', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-b:a', '192k', '-movflags', '+faststart', output, ]; } /** Validate the audio length against the model's per-resolution cap. */ export function assertAudioWithinLimit(audioSec: number, resolution: LipsyncResolution): void { const limit = OMNIHUMAN_AUDIO_LIMITS[resolution]; if (audioSec > limit + 1e-3) { throw new Error( `lipsync: audio is ${audioSec.toFixed(1)}s but OmniHuman ${resolution} caps at ${limit}s. ` + `Use --resolution 720p (60s cap) or split/trim the audio.`, ); } } export interface RunLipsyncOptions { image: string; audio: string; output: string; prompt?: string; resolution?: LipsyncResolution; turbo?: boolean; /** Normalize target fps (default 24). */ normalizeFps?: number; /** Skip the CFR/even-dims normalize (keep the raw OmniHuman clip). */ skipNormalize?: boolean; dryRun?: boolean; apiKey?: string; env?: NodeJS.ProcessEnv; awaitOptions?: XskillAwaitOptions; // Injectable transports (default the real public-host + xskill + ffmpeg). uploadImage?: (path: string) => Promise; uploadAudio?: (path: string) => Promise; submit?: (modelId: string, params: Record) => Promise; awaitTask?: (taskId: string) => Promise; download?: (url: string, dest: string) => Promise; probeAudioSec?: (path: string) => Promise; normalize?: (input: string, output: string, fps: number) => Promise; } export interface RunLipsyncResult { output: string; resolution: LipsyncResolution; audioSec: number; normalized: boolean; dryRun: boolean; params?: OmniHumanParams; taskId?: string; imageUrl?: string; audioUrl?: string; outputUrl?: string; } /** * Execute one lip-sync: validate audio length → (upload image+audio → submit * OmniHuman → await → download → normalize). On `dryRun`, returns the plan * (resolution, audio length, params) without uploading or spending. */ export async function runLipsync(opts: RunLipsyncOptions): Promise { const resolution = opts.resolution ?? '1080p'; const fps = opts.normalizeFps ?? 24; const probeAudioSec = opts.probeAudioSec ?? (async (p) => (await probeMedia(p)).durationSeconds ?? 0); const audioSec = await probeAudioSec(opts.audio); if (!audioSec) throw new Error(`lipsync: could not determine the duration of ${opts.audio}.`); assertAudioWithinLimit(audioSec, resolution); if (opts.dryRun) { const params = buildOmniHumanParams({ imageUrl: '', audioUrl: '', resolution, ...(opts.prompt ? { prompt: opts.prompt } : {}), ...(opts.turbo ? { turbo: true } : {}) }); return { output: opts.output, resolution, audioSec, normalized: !opts.skipNormalize, dryRun: true, params }; } const uploadImage = opts.uploadImage ?? (async (p) => (await hostPublicUrl(p, { ...(opts.env ? { env: opts.env } : {}) })).url); const uploadAudio = opts.uploadAudio ?? (async (p) => (await hostPublicUrl(p, { ...(opts.env ? { env: opts.env } : {}) })).url); const submit = opts.submit ?? ((modelId, p) => xskillSubmit(modelId, p, { ...(opts.apiKey ? { apiKey: opts.apiKey } : {}), ...(opts.env ? { env: opts.env } : {}) })); const awaitTask = opts.awaitTask ?? (async (taskId: string): Promise => { const done = await xskillAwait(taskId, { ...(opts.apiKey ? { apiKey: opts.apiKey } : {}), ...(opts.env ? { env: opts.env } : {}), ...opts.awaitOptions }); return done.videoUrl; }); const download = opts.download ?? ((url, dest) => xskillDownload(url, dest)); const normalize = opts.normalize ?? (async (input, output, f) => { await runFfmpeg(buildLipsyncNormalizeArgs(input, output, f)); }); const imageUrl = await uploadImage(opts.image); const audioUrl = await uploadAudio(opts.audio); const params = buildOmniHumanParams({ imageUrl, audioUrl, resolution, ...(opts.prompt ? { prompt: opts.prompt } : {}), ...(opts.turbo ? { turbo: true } : {}) }); const taskId = await submit(OMNIHUMAN_MODEL_ID, params as unknown as Record); const outputUrl = await awaitTask(taskId); const dir = await mkdtemp(join(tmpdir(), 'vclaw-lipsync-')); try { if (opts.skipNormalize) { await download(outputUrl, opts.output); return { output: opts.output, resolution, audioSec, normalized: false, dryRun: false, params, taskId, imageUrl, audioUrl, outputUrl }; } const raw = join(dir, 'omnihuman-raw.mp4'); await download(outputUrl, raw); try { await normalize(raw, opts.output, fps); } catch (err) { // The raw download is PAID OmniHuman output — salvage it before the // finally-cleanup deletes the tmpdir, and surface every recovery // identifier (salvaged path, task id, output URL) in the error. const salvaged = `${opts.output}.raw.mp4`; let salvageNote = `the raw output is salvaged at ${salvaged}`; try { await copyFile(raw, salvaged); } catch { salvageNote = `salvaging the raw output failed — re-download it from the output URL`; } throw new Error( `lipsync: normalize failed after a successful OmniHuman render; ${salvageNote} ` + `(task ${taskId}, url ${outputUrl}): ${(err as Error).message}`, ); } return { output: opts.output, resolution, audioSec, normalized: true, dryRun: false, params, taskId, imageUrl, audioUrl, outputUrl }; } finally { await rm(dir, { recursive: true, force: true }); } }