/** * Avatar-host layout (Layout D) — per-take host base clip generation. * * The `avatar-host` layout replaces the original speaker with an identity-locked * character host that SPEAKS each take's line. For each take we generate a fresh * host clip with **omni-flash R2V + native voice**: * * character reference image (R2V `ingredients:`) + the take's line + a voice * → omni-flash generates the audio AND the lip-synced video together * * Because Omni produces the speech and the mouth movement jointly, the avatar is * genuinely lip-synced (no separate TTS / no muxing / no lip-sync model). The * host clips are then stitched into the avatar reel and captioned via the local * renderer. Unlike the V2V "add-overlay" edit, R2V generation is NOT * moderation-blocked. * * Each step carries the take's spoken `text`; the voice + character reference are * supplied by the generator (see `avatar-host-transport.ts`). * * Credit safety is structural: every side effect (the go-bananas character * generation + the Veo I2V render) is delegated to an injectable * {@link MotionOverlayHostGenerator}. The CLI supplies a generator that shells the * real go-bananas + Veo I2V path; tests inject a fake, so the orchestrator runs * offline with NO real generation and NO spend. * * Dry vs execute mirrors {@link runMotionOverlay}: default (`confirmSpend` falsey) * enumerates the host clips that WOULD be generated without invoking the * generator; only `confirmSpend: true` actually drives it. */ import { join } from 'node:path'; import { takeStem } from './plan.js'; import type { MotionOverlayPlan, MotionTake, TakeSplit } from './types.js'; export class MotionOverlayAvatarError extends Error { constructor( public readonly code: string, message: string, ) { super(message); this.name = 'MotionOverlayAvatarError'; } } /** The subdir under the work folder where generated host base clips land. */ export const HOST_DIRNAME = 'host'; /** * A parsed `--gb-character` value: the locked go-bananas character that hosts the * reel. The `id` is the go-bananas character id; `name` is the human label used in * the generation prompt + manifest provenance. */ export interface GbCharacter { name: string; id: number; } /** * Parse a `--gb-character` flag value of the form `Name:ID` (e.g. `Clawbot:291`). * The name may contain spaces; only the FINAL `:` separates name from id so names * like `Dr. Vox:97` parse correctly. Pure. * * @throws {@link MotionOverlayAvatarError} (`gb_character_malformed`) on a missing * separator, an empty name, or a non-numeric / non-positive id. */ export function parseGbCharacter(raw: string): GbCharacter { const value = raw.trim(); const sep = value.lastIndexOf(':'); if (sep <= 0 || sep === value.length - 1) { throw new MotionOverlayAvatarError( 'gb_character_malformed', `--gb-character must be of the form Name:ID (e.g. Clawbot:291), got: ${JSON.stringify(raw)}`, ); } const name = value.slice(0, sep).trim(); const idRaw = value.slice(sep + 1).trim(); const id = Number(idRaw); if (!name) { throw new MotionOverlayAvatarError( 'gb_character_malformed', `--gb-character is missing a character name before ':', got: ${JSON.stringify(raw)}`, ); } if (!Number.isInteger(id) || id <= 0) { throw new MotionOverlayAvatarError( 'gb_character_malformed', `--gb-character id must be a positive integer, got: ${JSON.stringify(idRaw)}`, ); } return { name, id }; } /** Inputs for a single take's host clip generation. */ export interface MotionOverlayHostStep { index: number; /** The locked host character (parsed from --gb-character). */ character: GbCharacter; /** The line the host SPEAKS for this take (omni-flash generates the voice). */ text: string; /** The take this host clip is for (for prompt/anatomy context). */ anatomy: MotionTake['anatomy']; /** Target clip duration in seconds (matches the take). */ durationSeconds: number; /** Absolute path the generated host clip should be written to. */ outputPath: string; } /** * Injectable host generator — the single seam through which the go-bananas * character generation + Veo I2V render flow. The CLI supplies a generator that * shells the real path (go-bananas generate_with_character → Veo I2V startImage → * download host clip to `step.outputPath`). Tests inject a fake so the * orchestrator runs offline with no real generation. */ export interface MotionOverlayHostGenerator { /** * Generate one take's host base clip. Resolves once the host clip exists at * `step.outputPath`. */ generateHostClip(step: MotionOverlayHostStep): Promise; } export interface RunAvatarHostOptions { /** The injectable host generator (CLI shells real path; tests inject a fake). */ generator: MotionOverlayHostGenerator; /** Per-take spoken line (take index → text) the host should say. Missing → ''. */ textByIndex?: Record; /** * Permit the credit-spending go-bananas + Veo I2V generation to RUN. Default * false → dry plan (the generator is never invoked); the result enumerates the * host clips that WOULD be generated. */ confirmSpend?: boolean; } /** What happened (or would happen) to one take's host base clip. */ export interface MotionOverlayHostResult { index: number; anatomy: MotionTake['anatomy']; /** Absolute path to the generated host base clip (the V2V base for this take). */ hostClip: string; /** generated = the generator was invoked; planned = dry (not invoked). */ status: 'generated' | 'planned'; } export interface RunAvatarHostReport { schemaVersion: 1; /** True only when the generator actually ran (confirmSpend). */ generated: boolean; character: GbCharacter; hosts: MotionOverlayHostResult[]; /** * Per-take map: take index → absolute host clip path. This is what the V2V * execute path consumes as the base footage override. */ hostBaseByIndex: Record; } /** * Reconstruct a take's stem (e.g. `take-01_0s-10s`) from its manifest fields so * the host clip sits in `host/` with a matching name. Same {@link takeStem} the * planner used. */ function takeStemOf(take: MotionTake): string { const split: TakeSplit = { index: take.index, start: take.start, end: take.end, segmentIndices: [] }; return takeStem(split); } /** Absolute path the host base clip for a take is written to. */ export function hostClipPath(workdir: string, take: MotionTake): string { return join(workdir, HOST_DIRNAME, `${takeStemOf(take)}.mp4`); } /** * Generate (or dry-plan) the per-take host base clips for the avatar-host layout. * * Takes are processed strictly IN ORDER. Pure control flow — all generation I/O is * delegated to `opts.generator`. Never mutates the input plan. The returned * `hostBaseByIndex` map feeds {@link runMotionOverlay} so each take's V2V base is * the generated host clip instead of the source take. * * @throws {@link MotionOverlayAvatarError} (`motion_overlay_avatar_layout`) if the * plan's layout is not `avatar-host`, or (`motion_overlay_no_takes`) if there are * no takes. */ export async function runAvatarHostGeneration( plan: MotionOverlayPlan, character: GbCharacter, opts: RunAvatarHostOptions, ): Promise { if (plan.layout !== 'avatar-host') { throw new MotionOverlayAvatarError( 'motion_overlay_avatar_layout', `runAvatarHostGeneration: plan layout must be 'avatar-host', got: ${plan.layout}.`, ); } if (plan.takes.length === 0) { throw new MotionOverlayAvatarError( 'motion_overlay_no_takes', 'motion-overlay avatar-host: the plan has no takes to generate hosts for.', ); } const confirmSpend = opts.confirmSpend === true; const orderedTakes = [...plan.takes].sort((a, b) => a.index - b.index); const hosts: MotionOverlayHostResult[] = []; const hostBaseByIndex: Record = {}; for (const take of orderedTakes) { const hostClip = hostClipPath(plan.workdir, take); if (confirmSpend) { await opts.generator.generateHostClip({ index: take.index, character, text: opts.textByIndex?.[take.index] ?? '', anatomy: take.anatomy, durationSeconds: take.durationSeconds, outputPath: hostClip, }); } hosts.push({ index: take.index, anatomy: take.anatomy, hostClip, status: confirmSpend ? 'generated' : 'planned', }); hostBaseByIndex[take.index] = hostClip; } return { schemaVersion: 1, generated: confirmSpend, character, hosts, hostBaseByIndex, }; }