/** * Assemble-stage orchestrator (sub-slice 3i) — the capstone that ties the * post-execution assembly pipeline together behind `vclaw video assemble`. * * Runs the building blocks shipped in 3b–3h IN ORDER and collects an * `AssembleManifestEntry[]`: * 1. (optional) extractPdfSlides — PDF deck -> slide images (3c) * 2. (optional) generateTitleCard — branded title card (3d) * 3. animateSlide per slide — per-slide video segments (3e) * 4. generateTts per scene — per-scene narration audio (3b) * 5. (optional) generateMusic — background music bed (3f) * 6. stitch — final MP4 (3h) * 7. (advisory) qa-* checks — collected into warnings (3g) * * DRY-RUN is the tested surface. `assembleProject({ dryRun: true })` PLANS the * whole pipeline — every FFmpeg command + provider call is recorded into the * manifest/events WITHOUT executing anything or needing API keys. Real * execution (ffmpeg spawns + provider keys) is a HUMAN integration checkpoint, * explicitly out of scope for the unit tests (same boundary as 3e/3h). */ import { readFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { join, resolve as resolvePath } from 'node:path'; import type { VideoProjectWorkspace } from '../workspace.js'; import { artifactPathFor, writeArtifact } from '../artifact-store.js'; import { createAssembleReportArtifact, type AssembleReportArtifact } from '../artifacts.js'; import type { AssembleInput, AssembleManifestEntry, AssembleResult, } from './types.js'; import { extractPdfSlides } from './pdf.js'; import { generateTitleCard } from './title-card.js'; import { animateSlide, alignDurationToFrame } from './animate-slides.js'; import { generateTts } from './tts.js'; import { generateMusic } from './music.js'; import { stitch, type StitchInput, type ConcatStrategy } from './stitch.js'; import { buildAudioLayers, type AudioLayer } from './audio-mix-plan.js'; import { readProjectManifest } from '../workspace.js'; import { dialogueArtifactPathFor, type DialogueArtifact } from '../dialogue.js'; import { readSfxArtifact } from '../sfx.js'; import { runAssembleMediaQc } from './media-qc.js'; import { lintDialogue } from './qa-dialogue-lint.js'; import { checkNarration } from './qa-narration.js'; import { checkImageFilter } from './qa-image-filter.js'; /** Default per-scene narration duration (sec) used when no probe is available. */ const DEFAULT_SCENE_DURATION_SEC = 5; /** * Music-bed volume for a SELECTED soundtrack auto-attached in clip-stitch mode. * Deliberately far above the 0.05 dialogue-bed default: here the soundtrack IS * the score (sidechain-ducked under the narration layer when one exists). 0.55 * is the value shipped on the hermes-do-launch film (2026-07-04), whose finish * had to be hand-rolled in ffmpeg for want of exactly this wiring. */ const CLIP_STITCH_SOUNDTRACK_VOLUME = 0.55; /** Storyboard scene shape (subset we consume here). */ interface StoryboardScene { sceneIndex: number; description: string; dialogue?: string; durationSeconds?: number; scenePrompt?: { imagePrompt?: string }; /** Narrative color-language state = the post grade id applied to this scene's segment. */ colorState?: string; } interface StoryboardArtifactShape { projectSlug?: string; scenes?: StoryboardScene[]; } /** * The assemble-relevant knobs read from the brand-profile.json. The shipped * brand-profile.schema.json is presenter-routing focused (presenterName, * characterId, voiceId, intro/outro assets …); the assemble stage reads what it * needs from it loosely and falls back to defaults for anything absent. Extra * assemble fields (deck/music/concat) are honored if present without failing * brand-profile validation, since they live alongside the required routing * fields. */ interface BrandProfileForAssemble { presenterName?: string; voiceId?: string; introAsset?: string; outroAsset?: string; /** Optional PDF deck to rasterize into slides. */ deckPdf?: string; /** Optional title-card config. */ titleCard?: { title: string; subtitle?: string; background?: string }; /** Optional background-music config (the nex-brand knob). */ music?: { enabled?: boolean; prompt?: string; durationSec?: number; volume?: number }; /** Concat strategy override (bunty -> demuxer/auto, nex -> filter). */ concatStrategy?: ConcatStrategy; /** Intro/outro pre-encoded segment paths, if the brand provides them. */ introSegments?: string[]; outroSegments?: string[]; } async function loadStoryboard( workspace: VideoProjectWorkspace, ): Promise { const storyboardPath = artifactPathFor(workspace, 'storyboard'); if (!existsSync(storyboardPath)) return []; const parsed = JSON.parse(await readFile(storyboardPath, 'utf-8')) as StoryboardArtifactShape; const scenes = parsed.scenes ?? []; return [...scenes].sort((a, b) => a.sceneIndex - b.sceneIndex); } async function loadBrandProfile( brandProfilePath?: string, ): Promise { if (!brandProfilePath) return undefined; if (!existsSync(brandProfilePath)) return undefined; return JSON.parse(await readFile(brandProfilePath, 'utf-8')) as BrandProfileForAssemble; } /** Narration text for a scene: explicit dialogue, else the description. */ function narrationFor(scene: StoryboardScene): string { return (scene.dialogue ?? scene.description ?? '').trim(); } /** * Discover the project's dialogue + sfx audio clips (project-relative paths in * the dialogue.json / sfx.json artifacts) and resolve them to absolute, * on-disk-verified paths for the stitch audio mix. * * Presence-driven and additive: a project with NO dialogue.json and NO * sfx.json (or whose clip files are missing) yields empty arrays, so the * caller leaves `stitchInput.audioLayers` unset → byte-identical legacy stitch. * Missing clip files are skipped (existsSync-guarded) rather than failing. */ async function discoverDialogueSfxClips( workspaceRoot: string, slug: string, projectDir: string, ): Promise<{ dialoguePaths: string[]; sfxPaths: string[] }> { const dialoguePaths: string[] = []; const sfxPaths: string[] = []; // Dialogue: dialogue.json → turns[].path (project-relative). const dialoguePath = dialogueArtifactPathFor(workspaceRoot, slug); if (existsSync(dialoguePath)) { try { const artifact = JSON.parse(await readFile(dialoguePath, 'utf-8')) as DialogueArtifact; for (const turn of artifact.turns ?? []) { if (!turn.path) continue; const abs = resolvePath(projectDir, turn.path); if (existsSync(abs)) dialoguePaths.push(abs); } } catch { // Malformed dialogue.json → treat as absent (no layers). Non-fatal. } } // SFX: sfx.json → clips[].path (project-relative). try { const sfx = await readSfxArtifact(workspaceRoot, slug); for (const clip of sfx?.clips ?? []) { if (!clip.path) continue; const abs = resolvePath(projectDir, clip.path); if (existsSync(abs)) sfxPaths.push(abs); } } catch { // Malformed sfx.json → treat as absent. Non-fatal. } return { dialoguePaths, sfxPaths }; } /** * Discover the per-scene rendered clips `vclaw video execute` downloads to * `outputs/scene-.mp4`. Returns the found clips (in storyboard * order) and the scene indices that are missing a clip. Used by clip-stitch * mode (`fromRenderedClips`). */ function discoverRenderedClips( projectDir: string, scenes: StoryboardScene[], ): { found: Array<{ scene: StoryboardScene; path: string }>; missing: number[] } { const outputsDir = join(projectDir, 'outputs'); const found: Array<{ scene: StoryboardScene; path: string }> = []; const missing: number[] = []; for (const scene of scenes) { const clipPath = join(outputsDir, `scene-${scene.sceneIndex}.mp4`); if (existsSync(clipPath)) found.push({ scene, path: clipPath }); else missing.push(scene.sceneIndex); } return { found, missing }; } /** * Orchestrate the assemble pipeline. Returns an `AssembleResult` whose * `manifest` records each produced (or planned, on dry-run) asset in pipeline * order, `events` is a human-readable step log, and `warnings` collects the * advisory QA findings. * * On `dryRun`, every step is PLANNED (manifest entries + events) but nothing is * generated and no API key is required. */ export async function assembleProject(input: AssembleInput): Promise { const { workspace, brandProfilePath, ffmpegBin } = input; const dryRun = input.dryRun ?? false; const fromClips = input.fromRenderedClips ?? false; const manifest: AssembleManifestEntry[] = []; const events: string[] = []; const warnings: string[] = []; const scenes = await loadStoryboard(workspace); const brand = await loadBrandProfile(brandProfilePath); const assembleDir = join(workspace.projectDir, 'assemble'); const slidesDir = join(assembleDir, 'slides'); const audioDir = join(assembleDir, 'audio'); const segmentsDir = join(assembleDir, 'segments'); const outputPath = join(workspace.projectDir, 'outputs', 'final.mp4'); // --- Step 1: PDF slide extraction (optional) ------------------------------- // Skipped entirely in clip-stitch mode (segments are rendered clips, not slides). let slidePaths: string[] = []; if (!fromClips) { if (brand?.deckPdf) { const pdfPath = resolvePath(workspace.projectDir, brand.deckPdf); events.push(`pdf: extract slides from ${brand.deckPdf}`); if (dryRun) { // Plan one slide per scene as the dry-run estimate (no PDF parse). slidePaths = scenes.map((s) => join(slidesDir, `slide_${String(s.sceneIndex).padStart(3, '0')}.png`)); } else { const pdf = await extractPdfSlides({ pdfPath, outputDir: slidesDir }); slidePaths = pdf.pages.map((p) => p.path); } } else { // No deck: each scene's slide is its produced image asset (placeholder path). slidePaths = scenes.map((s) => join(slidesDir, `slide_${String(s.sceneIndex).padStart(3, '0')}.png`)); } } // --- Step 2: title card (optional) ----------------------------------------- if (brand?.titleCard) { const titleCardPath = join(assembleDir, 'title-card.png'); events.push(`title-card: "${brand.titleCard.title}"`); const tc = await generateTitleCard({ title: brand.titleCard.title, subtitle: brand.titleCard.subtitle, background: brand.titleCard.background, outputPath: titleCardPath, dryRun, }); manifest.push({ kind: 'title-card', path: dryRun ? titleCardPath : tc.path, durationMs: 0, sizeBytes: 0, generator: 'assemble/title-card.ts', }); } // --- Steps 3+4: body segments ---------------------------------------------- // `segmentScenes` is the subset of scenes that actually produced a body // segment, in order — it aligns the narrative color grade in step 6 (in clip // mode, scenes missing a rendered clip are dropped from both). const segmentPaths: string[] = []; const segmentScenes: StoryboardScene[] = []; if (fromClips) { // Clip-stitch mode: body segments are the per-scene rendered clips from // `vclaw video execute` (outputs/scene-.mp4). Their native audio is kept, // so there is NO TTS narration step. const { found, missing } = discoverRenderedClips(workspace.projectDir, scenes); if (missing.length > 0) { warnings.push( `clip-stitch: ${missing.length} scene(s) missing a rendered clip at outputs/scene-.mp4: [${missing.join(', ')}]`, ); } if (found.length === 0) { warnings.push( 'clip-stitch: no rendered clips found at outputs/scene-.mp4 — run `vclaw video execute` first.', ); } for (const { scene, path } of found) { events.push(`clip: scene ${scene.sceneIndex} -> ${path}`); segmentPaths.push(path); segmentScenes.push(scene); manifest.push({ kind: 'rendered-clip', path, durationMs: 0, sceneIndex: scene.sceneIndex, sizeBytes: 0, generator: 'vclaw video execute', }); } } else { // --- Step 4 (computed first): per-scene narration (TTS) ------------------ // TTS is needed to drive the per-slide segment durations in step 3, so we // plan it before animation even though the canonical pipeline lists it after. const ttsSegments = scenes.map((s) => ({ sceneIndex: s.sceneIndex, text: narrationFor(s) })); const narrationDurationMsByScene = new Map(); if (ttsSegments.length > 0 && (brand?.voiceId || dryRun)) { events.push(`tts: ${ttsSegments.length} scene narration(s)`); const tts = await generateTts({ segments: ttsSegments, voiceId: brand?.voiceId ?? 'dry-run-voice', outputDir: audioDir, dryRun, }); for (const scene of tts.scenes) { if (scene.durationMs > 0) { narrationDurationMsByScene.set(scene.sceneIndex, scene.durationMs); } manifest.push({ kind: 'narration', path: scene.path, durationMs: scene.durationMs, sceneIndex: scene.sceneIndex, sizeBytes: scene.sizeBytes, generator: 'assemble/tts.ts', }); } // Real runs also surface tts.manifest entries (already shaped); merge any // not already represented (defensive — dry-run returns an empty manifest). } else if (ttsSegments.length > 0) { warnings.push('tts skipped: no voiceId in brand profile (real run requires one).'); } // --- Step 3: per-slide animation -> segments ---------------------------- for (const scene of scenes) { const slidePath = slidePaths[scene.sceneIndex] ?? slidePaths[scenes.indexOf(scene)] ?? ''; const ttsPath = join(audioDir, `scene_${String(scene.sceneIndex).padStart(3, '0')}.mp3`); const segmentPath = join(segmentsDir, `seg_slide_${String(scene.sceneIndex).padStart(3, '0')}.mp4`); const narrationDurationMs = narrationDurationMsByScene.get(scene.sceneIndex); const durationSec = narrationDurationMs && narrationDurationMs > 0 ? narrationDurationMs / 1000 : scene.durationSeconds ?? DEFAULT_SCENE_DURATION_SEC; events.push(`animate: scene ${scene.sceneIndex} -> ${segmentPath}`); if (dryRun) { // Plan the segment without spawning ffmpeg. segmentPaths.push(segmentPath); segmentScenes.push(scene); manifest.push({ kind: 'slide-animation', path: segmentPath, durationMs: Math.round(alignDurationToFrame(durationSec) * 1000), sceneIndex: scene.sceneIndex, sizeBytes: 0, generator: 'assemble/animate-slides.ts', }); } else { const seg = await animateSlide( { slidePath, ttsPath, outputPath: segmentPath, durationSec, slideNum: scenes.indexOf(scene) + 1, numSlides: scenes.length, }, { ffmpegBin }, ); segmentPaths.push(seg.path); segmentScenes.push(scene); manifest.push({ kind: 'slide-animation', path: seg.path, durationMs: seg.durationMs, sceneIndex: scene.sceneIndex, sizeBytes: 0, generator: 'assemble/animate-slides.ts', }); } } } // --- Step 5: background music (optional) ----------------------------------- let musicPath: string | undefined; if (brand?.music?.enabled) { musicPath = join(assembleDir, 'music.mp3'); const prompt = brand.music.prompt ?? 'Soft ambient background bed, instrumental.'; events.push('music: generate background bed'); const music = await generateMusic({ prompt, durationSec: brand.music.durationSec, outputPath: musicPath, dryRun, }); manifest.push({ kind: 'music', path: music.path, durationMs: music.durationMs, sizeBytes: 0, generator: 'assemble/music.ts', }); } // --- Step 6: stitch -> final MP4 ------------------------------------------- let finalOutputPath = outputPath; if (segmentPaths.length > 0) { // Narrative color language: each scene's colorState becomes the per-segment // grade. Align it to the ordered segments (intro + body + outro): intro/outro // carry no grade. Omitted entirely when no scene is tagged, so default output // is unchanged. segmentScenes is one-per-body-segment in the same (sorted) // order — in clip mode scenes missing a clip are already dropped from both. const bodyGradeIds = segmentScenes.map((s) => s.colorState); const segmentGradeIds = bodyGradeIds.some((g) => typeof g === 'string' && g !== '') ? [ ...new Array(brand?.introSegments?.length ?? 0).fill(undefined), ...bodyGradeIds, ...new Array(brand?.outroSegments?.length ?? 0).fill(undefined), ] : undefined; // Discover dialogue + sfx clips and mix them (with the existing music bed) // as GLOBAL audio layers at the stitch step. Presence-driven & additive: // a project without dialogue.json/sfx.json (or with missing clip files) // yields no extra layers, leaving `audioLayers` unset → byte-identical // legacy stitch (music-only buildMusicMixArgs, or no-audio). // // Clip-stitch mode keeps each clip's own audio, so the dialogue/sfx // auto-layers are NOT applied (the music bed remains the only optional // extra layer, mixed under the clip audio at the stitch step). const { dialoguePaths, sfxPaths } = fromClips ? { dialoguePaths: [] as string[], sfxPaths: [] as string[] } : await discoverDialogueSfxClips( workspace.root, workspace.slug, workspace.projectDir, ); // Clip-stitch audio auto-attach (presence-driven, additive): a project that // ran `vclaw video narrate` and/or selected a soundtrack (`soundtrack // --select`, which writes the manifest `soundtrack` field) gets them mixed // at the stitch step — narration as a global voice layer over the kept clip // audio, the selected soundtrack as the score bed sidechain-ducked under // it. Neither artifact present → no layers → byte-identical legacy // clip-stitch. (Production evidence: the hermes-do-launch film, 2026-07-04, // had to hand-roll exactly this mix in raw ffmpeg.) let clipAudioLayers: AudioLayer[] | undefined; let clipNarrationAttached = false; let clipSoundtrackAttached = false; if (fromClips) { const narrationPath = join(workspace.projectDir, 'artifacts', 'audio', 'narration.mp3'); const haveNarration = existsSync(narrationPath); const projectManifest = await readProjectManifest(workspace); const soundtrackRel = projectManifest?.soundtrack ?? undefined; const soundtrackAbs = soundtrackRel ? resolvePath(workspace.projectDir, soundtrackRel) : undefined; const haveSoundtrack = Boolean(soundtrackAbs && existsSync(soundtrackAbs)); // brand.music generates its own bed via `input.music` — never double-bed. const attachSoundtrack = haveSoundtrack && !brand?.music?.enabled; if (haveSoundtrack && brand?.music?.enabled) { warnings.push( 'clip-stitch: both a selected soundtrack (manifest `soundtrack`) and brand.music are present — using the brand music bed; the selected soundtrack was NOT attached.', ); } if (haveNarration || attachSoundtrack) { clipAudioLayers = buildAudioLayers( { ...(attachSoundtrack && soundtrackAbs ? { musicPath: soundtrackAbs } : {}), ...(haveNarration ? { narrationPath } : {}), }, { musicVolume: CLIP_STITCH_SOUNDTRACK_VOLUME }, ); clipNarrationAttached = haveNarration; clipSoundtrackAttached = attachSoundtrack; } } const audioLayers = fromClips ? clipAudioLayers : dialoguePaths.length > 0 || sfxPaths.length > 0 ? buildAudioLayers({ dialoguePaths, sfxPaths }) : undefined; // Voice-forward the music bed by default whenever per-scene narration was // produced (the manifest carries `narration` entries): loudnorm the voice, // sidechain-duck the music under it, and limit the output — so narration is // never buried beneath the bed. No narration → byte-identical legacy bed mix. const hasNarration = manifest.some((m) => m.kind === 'narration'); const stitchInput: StitchInput = { segments: segmentPaths, intro: brand?.introSegments, outro: brand?.outroSegments, outputPath, concatStrategy: brand?.concatStrategy, ...(segmentGradeIds ? { segmentGradeIds } : {}), ...(musicPath ? { music: { trackPath: musicPath, volume: brand?.music?.volume, ...(hasNarration ? { voiceForward: true } : {}) } } : {}), ...(audioLayers ? { audioLayers } : {}), // Duck the soundtrack under the narration in the multi-layer mix — the // clip-stitch analogue of the slide path's voiceForward. Only meaningful // when both a voice layer and a music layer are present. ...(clipNarrationAttached && (clipSoundtrackAttached || Boolean(musicPath)) ? { duckMusicUnderVoice: true } : {}), }; if (audioLayers) { events.push( fromClips ? `audio-mix: clip-stitch auto-attach — ${clipNarrationAttached ? 'narration' : ''}${clipNarrationAttached && clipSoundtrackAttached ? ' + ' : ''}${clipSoundtrackAttached ? 'selected soundtrack (ducked under voice)' : ''} mixed over clip audio` : `audio-mix: ${dialoguePaths.length} dialogue + ${sfxPaths.length} sfx clip(s) mixed as global layers` + (musicPath ? ' (+music bed)' : ''), ); } events.push( `stitch: ${segmentPaths.length} segment(s) -> ${outputPath}` + (musicPath ? ' (+music)' : '') + (audioLayers ? ' (+dialogue/sfx)' : ''), ); const stitched = await stitch(stitchInput, { dryRun, ffmpegBin }); finalOutputPath = stitched.outputPath; for (const step of stitched.plan) { events.push(`stitch.plan: ${step.kind} -> ${step.outputPath}`); } manifest.push({ kind: 'final-video', path: stitched.outputPath, durationMs: stitched.durationMs, sizeBytes: 0, generator: 'assemble/stitch.ts', }); } else if (!fromClips) { // Clip mode already pushed a precise clip-stitch warning above. warnings.push('stitch skipped: no slide segments (empty storyboard).'); } const qc = dryRun ? undefined : await runAssembleMediaQc({ manifest, outputPath: finalOutputPath }); if (qc) { for (const issue of qc.issues) { warnings.push(`qc.${issue.code}[${issue.scope}]: ${issue.message}`); } } // --- Step 7: advisory QA --------------------------------------------------- if (scenes.length > 0) { const dialogueResult = lintDialogue({ segments: scenes.map((s) => ({ sceneIndex: s.sceneIndex, text: narrationFor(s) })), }); for (const w of dialogueResult.warnings) { warnings.push(`qa.dialogue[scene ${w.sceneIndex}/${w.rule}]: ${w.message}`); } const narrationResult = checkNarration({ scenes: scenes.map((s) => ({ sceneIndex: s.sceneIndex, narration: narrationFor(s) })), slideCount: slidePaths.length || undefined, }); for (const w of narrationResult.warnings) { warnings.push(`qa.narration[scene ${w.sceneIndex}/${w.rule}]: ${w.message}`); } const imageFilterResult = checkImageFilter({ candidates: scenes .filter((s) => s.scenePrompt?.imagePrompt) .map((s) => ({ sceneIndex: s.sceneIndex, prompt: s.scenePrompt!.imagePrompt! })), }); for (const w of imageFilterResult.warnings) { warnings.push(`qa.image-filter[scene ${w.sceneIndex}/${w.verdict}]: ${w.message}`); } } const status: AssembleResult['status'] = dryRun ? 'dry-run' : warnings.length > 0 ? 'partial' : 'complete'; return { status, outputPath: finalOutputPath, manifest, events, warnings, ...(qc ? { qc } : {}), }; } /** * Persist an `assemble-report.json` artifact for a completed (or dry-run) * assemble pass via the TYPED `writeArtifact` helper (so the artifact is no * longer an "alternate writer" — it's schema-covered). Validates against * schemas/video/artifacts/assemble-report.schema.json. */ export async function writeAssembleReport( workspace: VideoProjectWorkspace, result: AssembleResult, brandProfilePath?: string, ): Promise<{ artifactPath: string; report: AssembleReportArtifact }> { const report = createAssembleReportArtifact({ projectSlug: workspace.slug, status: result.status, brandProfile: brandProfilePath ?? null, outputPath: result.outputPath, manifest: result.manifest.map((entry) => ({ kind: entry.kind, path: entry.path, durationMs: entry.durationMs, ...(entry.sceneIndex !== undefined ? { sceneIndex: entry.sceneIndex } : {}), sizeBytes: entry.sizeBytes, generator: entry.generator, })), warnings: result.warnings, events: result.events, ...(result.qc ? { qc: result.qc } : {}), }); const artifactPath = await writeArtifact(workspace, 'assemble-report', report); return { artifactPath, report }; }