/** * Stitch keystone for the assemble stage (sub-slice 3h). * * Source of truth (ported VERBATIM): * - `skills/video-replicator/scripts/stitch_bunty.py` — the bunty stitch. Uses * the concat **demuxer** path (`ffmpeg -y -f concat -safe 0 -i concat.txt * -c copy output`) — a single ffmpeg invocation regardless of segment count, * chosen deliberately so it survives the sandbox's per-session FFmpeg limit. * - `skills/video-replicator/scripts/ffmpeg_wrapper.py:concat_via_filter` — the * concat **filter** fallback (`-filter_complex * "[0:v][0:a]...concat=n=N:v=1:a=1[outv][outa]"` + re-encode). Used for 8+ * segments where demuxer-accumulated AV drift across boundaries matters, OR * when a segment has incompatible codec params and the demuxer rejects it. * - `skills/video-replicator/scripts/assembly_utils.py:add_background_music` — * the music-bed mix (loops the music under the narration at low volume with a * tail fade-out, via amix). * * bunty (stitch_bunty.py) and nex (nex_assemble.py) collapse into ONE * parameterized stitch driven by brand-profile-derived knobs: * - bunty: pre-encoded segments → demuxer concat, no music, intro/outro by * lip-sync scene segments. (`_concat_via_demuxer`, demuxer-first with a * filter fallback.) * - nex: normalized segments → filter concat (drift-free for 8+), optional * background-music bed, optional title-card prepend. (`concat_via_filter` * + `add_background_music`.) * These differences become StitchInput fields: `concatStrategy` * (demuxer | filter | auto), `intro` / `outro` segment paths, and the optional * `music` block (track + volume + tail fade). The demuxer-vs-filter selection in * `auto` mode flips at `FILTER_FALLBACK_SEGMENT_THRESHOLD` segments — the real * sandbox-survival + drift lesson from the Python. * * AV-drift note (stitch_bunty.py ~L471-479): demuxer `-c copy` preserves exact * packet timing and accumulates no re-encode drift, BUT it requires every * segment to share encoding params. The per-segment AV-lock from 3e * (1280×720@24, H.264 libx264 preset-fast crf20, AAC 44100 stereo) guarantees * that, so the demuxer path is the primary one. Filter concat re-encodes (and so * can introduce its own drift) but tolerates mismatched inputs — it is the * fallback. * * IMPORTANT — testing boundary (SAME as 3e): the PURE arg-builders * (`buildConcatDemuxerArgs`, `buildConcatFilterArgs`, `buildMusicMixArgs`) are * the unit-tested surface (arg-shape only). `stitch` actually spawns ffmpeg and * writes the concat list; verifying that the final MP4 looks/sounds right on * real media is a HUMAN integration checkpoint, explicitly OUT OF SCOPE for the * unit tests. Tests use the dry-run path and never run ffmpeg or require media. */ import { writeFile, mkdir, stat } from 'node:fs/promises'; import { dirname, resolve as resolvePath } from 'node:path'; import { runFfmpeg, ffprobeDuration, isValidMp4, trimTailArgs, letterboxFilter, type RunFfmpegOptions } from './ffmpeg.js'; import { probeMedia } from '../final-media.js'; import { VclawError } from '../errors.js'; /** * Segment-count threshold at which `auto` strategy switches from the * concat demuxer to the concat filter. Mirrors the Python lesson that the * demuxer accumulates audible AV drift across "8+ segments" * (ffmpeg_wrapper.concat_via_filter docstring). At or above this many segments * `auto` prefers the filter path. */ export const FILTER_FALLBACK_SEGMENT_THRESHOLD = 8; /** Default background-music mix level (nex_assemble.py --music-volume default 0.05). */ export const DEFAULT_MUSIC_VOLUME = 0.05; /** Default tail fade-out for the music bed in seconds (add_background_music default 3.0). */ export const DEFAULT_MUSIC_FADE_OUT_SEC = 3.0; /** AAC bitrate for re-encode paths (filter concat + music mix). Matches the Python "192k". */ export const STANDARD_AUDIO_BITRATE = '192k'; /** * Named post-production color grades → FFmpeg video-filter chains (WS9+). Applied * in the per-segment prep pass, so different segments can carry different grades — * which is exactly the reference advert's narrative color language (cool-steel * "normal ops" -> crimson "breach" -> electric-blue "resolution"). These are real * post transforms on the rendered footage (colorbalance/eq), NOT prompt hints. * `kodak-500t` approximates the Vision3 500T tungsten stock (cool shadows, warm * highlights, mild desat). For a true film LUT, pass a `.cube` via `gradeLut`. */ const GRADE_FILTERS: Record = { 'cool-steel': 'colorbalance=rs=-0.08:gs=-0.04:bs=0.12,eq=saturation=0.82:contrast=1.06', 'crimson-threat': 'colorbalance=rs=0.18:gs=-0.06:bs=-0.08,eq=saturation=1.05:contrast=1.12', 'electric-blue': 'colorbalance=rs=-0.06:gs=0.04:bs=0.18,eq=saturation=1.00:contrast=1.05', 'kodak-500t': 'colorbalance=bs=0.08:rh=0.05:bh=-0.05,eq=saturation=0.90:contrast=1.05', 'bleach-bypass': 'eq=saturation=0.50:contrast=1.32', 'desaturated': 'eq=saturation=0.70:contrast=1.08', 'teal-orange': 'colorbalance=rs=0.05:bs=0.10:rh=0.08:bh=-0.06,eq=saturation=1.08', }; /** Stable list of the named grade ids (for CLI validation / docs). */ export const GRADE_FILTER_IDS = Object.keys(GRADE_FILTERS); /** * Resolve a named grade id to its FFmpeg filter chain. Returns '' for an * undefined/empty/unknown id (the prep pass then applies no grade) — callers that * want a hard failure on an unknown id should validate against {@link GRADE_FILTER_IDS}. */ export function resolveGradeFilter(id: string | undefined): string { if (!id) return ''; return GRADE_FILTERS[id] ?? ''; } /** Which concat path to use. `auto` picks demuxer/filter by segment count. */ export type ConcatStrategy = 'demuxer' | 'filter' | 'auto'; export interface MusicMixSettings { /** Path to the background-music track. Looped under the narration. */ trackPath: string; /** Mix level for the music bed (0..1). Defaults to {@link DEFAULT_MUSIC_VOLUME}. */ volume?: number; /** Tail fade-out duration in seconds. Defaults to {@link DEFAULT_MUSIC_FADE_OUT_SEC}. */ fadeOutSec?: number; /** * Voice-forward mix (loudnorm the video's narration + sidechain-duck the music * under it + final limiter). Forwarded to {@link BuildMusicMixOptions.voiceForward}. * Default off → byte-identical legacy bed mix. */ voiceForward?: boolean; } export interface StitchInput { /** * Ordered body segment paths (the slide segments). These sit between the * optional intro and outro segments. From 3e every segment conforms to * 1280×720@24 / H.264 / AAC 44100 stereo, so demuxer concat is valid. */ segments: string[]; /** Optional ordered intro segment paths, prepended before `segments`. */ intro?: string[]; /** Optional ordered outro segment paths, appended after `segments`. */ outro?: string[]; /** Where the final stitched MP4 is written. */ outputPath: string; /** * Path for the concat-demuxer list file. Defaults to `concat.txt` alongside * the output. Only used by the demuxer path. */ concatListPath?: string; /** * Concat strategy. `auto` (default) uses the demuxer up to * {@link FILTER_FALLBACK_SEGMENT_THRESHOLD} segments, then the filter. * bunty maps to `auto`/`demuxer`; nex maps to `filter`. */ concatStrategy?: ConcatStrategy; /** Optional background-music bed (the nex-brand knob). Omit for bunty. */ music?: MusicMixSettings; /** * Optional EXTRA global audio layers (dialogue / sfx / additional voice) * mixed over the concatenated video at the stitch step, BEYOND the `music` * bed above. Presence-driven and additive: when this is empty/undefined the * stitch is byte-identical to legacy (the music-only {@link buildMusicMixArgs} * path, or the no-audio path). When non-empty, the music bed (if any) is * prepended as a `music` layer and the whole set is mixed via * {@link buildMultiLayerMixArgs}. See assemble.ts Step 6 (dialogue/sfx wiring). */ audioLayers?: AudioLayer[]; /** * Sidechain-duck every `music` layer under the narration/dialogue layers in * the MULTI-LAYER mix (`audioLayers` path). The multi-layer analogue of * `music.voiceForward`: without it a soundtrack bed sits at full level over * a narration layer. Only consulted when `audioLayers` is non-empty; unset → * byte-identical legacy multi-layer mix. */ duckMusicUnderVoice?: boolean; /** * Optional per-clip tail cut (WS9). When set, each ordered segment is * re-encoded with `-t ` before concat, dropping the dead / * freeze frames AI generators append. Omit for byte-identical legacy behavior. */ clipMaxSeconds?: number; /** * Optional letterbox normalization (WS9). When set (e.g. `2.39:1`), each * segment is scaled+padded onto the {@link letterboxCanvas} (default * 1280×720) before concat, producing cinematic bars. Omit to disable. */ letterboxRatio?: string; /** * Canvas for {@link letterboxRatio}. Defaults to the 3e segment standard * (1280×720) so normalized segments stay uniform for demuxer concat. */ letterboxCanvas?: { width: number; height: number }; /** * Path to a `.cube`/`.3dl` LUT applied to every segment via `lut3d` (e.g. a * Kodak Vision3 500T film-stock LUT). Omit to skip. Triggers the prep pass. */ gradeLut?: string; /** * Named color grade applied to every segment (see {@link GRADE_FILTER_IDS}). * A real post transform on the footage, not a prompt hint. Omit to skip. */ gradeId?: string; /** * Per-segment grade overrides, indexed to {@link orderedSegments} (intro + * body + outro). A defined entry overrides {@link gradeId} for that segment — * this is how the narrative color language is realized (e.g. cool-steel for the * calm scenes, crimson-threat for the breach, electric-blue for the resolution). */ segmentGradeIds?: Array; /** * Optional reading-holds (motion-comic). For each listed segment index (into * {@link orderedSegments}: intro+body+outro), a short held, gently-drifting * still of that segment's readable FIRST frame is prepended BEFORE its motion * plays — so viewers can read text-dense info panels (a cast roster, a strategy * map) before the animation takes over. Omit / empty `segments` → byte-identical * legacy (no holds). When any hold is present the concat uses the FILTER path * (re-encode) so the freshly-encoded hold clips concat cleanly with the segments. */ readingHold?: ReadingHoldSettings; } export interface ReadingHoldSettings { /** Hold duration in seconds before each designated segment. Default 3.5. */ holdSec?: number; /** Segment indices (into orderedSegments) that get a reading hold. Empty ⇒ no holds. */ segments: number[]; } export interface BuildReadingHoldOptions { /** Hold duration in seconds. Default 3.5. */ holdSec?: number; /** Output width. Default 1280. */ width?: number; /** Output height. Default 720. */ height?: number; /** Output fps. Default 24. */ fps?: number; } /** * Build ffmpeg args (PURE) that turn a video segment's FIRST frame into a short * held, gently-drifting still clip (a "reading hold"), with a silent stereo audio * track so it concats cleanly. `select=eq(n,0)` grabs frame 0; `zoompan` holds it * for `holdSec` with a faint slow zoom (alive, not frozen); encoded to the segment * standard (libx264 crf20 / AAC 44100 stereo @ fps). `-y` is prepended by runFfmpeg. */ export function buildReadingHoldArgs( segmentPath: string, outputPath: string, opts: BuildReadingHoldOptions = {}, ): string[] { const holdSec = opts.holdSec ?? 3.5; const width = opts.width ?? 1280; const height = opts.height ?? 720; const fps = opts.fps ?? 24; const frames = Math.round(holdSec * fps); // Oversample 2× so the slow zoom stays sharp, then output at the canvas size. const vf = `select=eq(n\\,0),scale=${width * 2}:${height * 2}:force_original_aspect_ratio=increase,` + `crop=${width * 2}:${height * 2},` + `zoompan=z='min(zoom+0.0002,1.03)':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=${frames}:s=${width}x${height}:fps=${fps},setsar=1`; return [ '-i', segmentPath, '-f', 'lavfi', '-t', String(holdSec), '-i', 'anullsrc=r=44100:cl=stereo', '-filter_complex', `[0:v]${vf}[v]`, '-map', '[v]', '-map', '1:a', '-t', String(holdSec), '-r', String(fps), '-c:v', 'libx264', '-preset', 'fast', '-crf', '20', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-b:a', STANDARD_AUDIO_BITRATE, '-ar', '44100', '-ac', '2', '-movflags', '+faststart', outputPath, ]; } export interface StitchPlannedStep { /** What this step does. */ kind: 'segment-prep' | 'reading-hold' | 'concat-demuxer' | 'concat-filter' | 'music-mix' | 'multi-layer-mix'; /** The ffmpeg args (everything after the binary and the auto-prepended `-y`). */ args: string[]; /** The output this step writes. */ outputPath: string; } export interface StitchResult { status: 'complete' | 'dry-run'; /** Final MP4 path. */ outputPath: string; /** Ordered list of every segment that went into the concat. */ orderedSegments: string[]; /** Which concat path was actually used. */ concatStrategy: 'demuxer' | 'filter'; /** Whether a music bed was mixed. */ music: boolean; /** The planned ffmpeg command sequence (always populated, incl. dry-run). */ plan: StitchPlannedStep[]; /** Final video duration in milliseconds (0 on dry-run — no probe). */ durationMs: number; } /** Assemble the full ordered segment list: intro + body + outro. */ export function orderedSegments(input: StitchInput): string[] { return [...(input.intro ?? []), ...input.segments, ...(input.outro ?? [])]; } /** * Choose the effective concat path. `demuxer`/`filter` are honored directly; * `auto` flips to the filter at {@link FILTER_FALLBACK_SEGMENT_THRESHOLD} * segments (the drift lesson from the Python). */ export function selectConcatStrategy( strategy: ConcatStrategy, segmentCount: number, ): 'demuxer' | 'filter' { if (strategy === 'demuxer' || strategy === 'filter') return strategy; return segmentCount >= FILTER_FALLBACK_SEGMENT_THRESHOLD ? 'filter' : 'demuxer'; } /** Render the concat-demuxer list file body: one `file ''` line per segment. */ export function buildConcatListContent(segments: string[]): string { // Mirrors stitch_bunty._concat_via_demuxer: file ''. // DEVIATION from the Python (which had this bug): single quotes in the path // are escaped ('\'') — an apostrophe anywhere in a parent directory name // would otherwise corrupt the concat list for every segment. return segments.map((seg) => `file '${resolvePath(seg).replace(/'/g, `'\\''`)}'\n`).join(''); } /** * Build the concat-DEMUXER ffmpeg args (PURE). The primary path. * * Ported VERBATIM from stitch_bunty._concat_via_demuxer (L240): * ffmpeg -y -f concat -safe 0 -i concat.txt -c copy output * (`-y` is prepended by `runFfmpeg`, so it is NOT included here.) * * A single ffmpeg invocation regardless of segment count — survives the * sandbox's per-session FFmpeg limit. Requires all segments to share encoding * params (true for 3e segments). */ export function buildConcatDemuxerArgs( _segments: string[], concatListPath: string, outputPath: string, ): string[] { return ['-f', 'concat', '-safe', '0', '-i', concatListPath, '-c', 'copy', outputPath]; } export interface BuildConcatFilterOptions { /** H.264 CRF (Python default 20). */ crf?: number; /** AAC bitrate (Python default "192k"). */ audioBitrate?: string; /** Audio sample rate Hz (Python default 44100). */ sampleRate?: number; /** Audio channel count (Python default 2). */ channels?: number; } /** * Build the concat-FILTER fallback ffmpeg args (PURE). For 8+ segments where * demuxer drift accumulates, or when a segment has incompatible codec params. * * Ported VERBATIM from ffmpeg_wrapper.concat_via_filter (L296-326): * -i f0 -i f1 ... -i f{n-1} * -filter_complex "[0:v][0:a][1:v][1:a]...concat=n=N:v=1:a=1[outv][outa]" * -map [outv] -map [outa] * -c:v libx264 -preset fast -crf 20 * -c:a aac -b:a 192k -ar 44100 -ac 2 * -movflags +faststart output */ export function buildConcatFilterArgs( segments: string[], outputPath: string, opts: BuildConcatFilterOptions = {}, ): string[] { const crf = opts.crf ?? 20; const audioBitrate = opts.audioBitrate ?? STANDARD_AUDIO_BITRATE; const sampleRate = opts.sampleRate ?? 44100; const channels = opts.channels ?? 2; const inputArgs: string[] = []; for (const seg of segments) { inputArgs.push('-i', seg); } // [0:v][0:a][1:v][1:a]...[n-1:v][n-1:a]concat=n=N:v=1:a=1[outv][outa] let filterInputs = ''; for (let i = 0; i < segments.length; i += 1) { filterInputs += `[${i}:v][${i}:a]`; } const filterComplex = `${filterInputs}concat=n=${segments.length}:v=1:a=1[outv][outa]`; return [ ...inputArgs, '-filter_complex', filterComplex, '-map', '[outv]', '-map', '[outa]', '-c:v', 'libx264', '-preset', 'fast', '-crf', String(crf), '-c:a', 'aac', '-b:a', audioBitrate, '-ar', String(sampleRate), '-ac', String(channels), '-movflags', '+faststart', outputPath, ]; } export interface BuildSegmentPrepOptions { /** Per-clip tail cut in seconds (WS9 trimTailArgs). Omit/<=0 disables the trim. */ clipMaxSeconds?: number; /** Letterbox target ratio label (WS9 letterboxFilter). Omit/'' disables the filter. */ letterboxRatio?: string; /** Path to a `.cube`/`.3dl` LUT applied via `lut3d` (e.g. a Kodak Vision3 500T LUT). Omit to skip. */ gradeLut?: string; /** Resolved FFmpeg grade filter chain (from {@link resolveGradeFilter}). Omit/'' to skip. */ gradeFilter?: string; /** Letterbox canvas width. Default 1280 (3e segment standard). */ width?: number; /** Letterbox canvas height. Default 720. */ height?: number; /** Output frame rate. Default 24. */ fps?: number; /** H.264 CRF. Default 20. */ crf?: number; /** AAC bitrate. Default {@link STANDARD_AUDIO_BITRATE}. */ audioBitrate?: string; /** Audio sample rate Hz. Default 44100. */ sampleRate?: number; /** Audio channels. Default 2. */ channels?: number; /** * Pad the audio track with trailing silence to exactly the video duration * (`apad` + `-shortest`), so the prepped segment is A/V-aligned and a demuxer * `-c copy` concat of such segments cannot accumulate drift (the "echo" / * overlapping-narration bug from the film builds). It also TRIMS audio that * runs past the video, preventing one scene's narration bleeding into the next. * Default true; `stitch()` auto-disables it for a segment with no audio stream * (probed on real runs — `apad` needs an audio source on stricter ffmpeg * builds). The re-encode already emits AAC, so this only adds the pad/shortest, * not a second pass. */ padAudioToVideoDuration?: boolean; } /** * Build the per-segment normalization ffmpeg args (PURE, WS9). Re-encodes one * clip to the 3e segment standard (1280×720@24 / libx264 preset-fast crf20 / AAC * 44100 stereo) while optionally trimming its tail ({@link trimTailArgs}) and * letterboxing it onto a canvas ({@link letterboxFilter}). Re-encoding to the * standard keeps the normalized segments uniform, so the demuxer `-c copy` concat * stays valid. By default the audio is also padded/trimmed to the video length * ({@link BuildSegmentPrepOptions.padAudioToVideoDuration}) so segments are * A/V-aligned and a `-c copy` concat cannot accumulate drift. `-t` is placed just * before the output path so it caps the OUTPUT. */ export function buildSegmentPrepArgs( segment: string, outputPath: string, opts: BuildSegmentPrepOptions = {}, ): string[] { const width = opts.width ?? 1280; const height = opts.height ?? 720; // Compose the vf chain in deterministic order: letterbox (geometry) → LUT // (film-stock emulation) → named grade (state color). Each is optional; empties // are dropped, so default behavior is no `-vf` at all. const vf = [ letterboxFilter(opts.letterboxRatio, width, height), opts.gradeLut ? `lut3d=${opts.gradeLut}` : '', opts.gradeFilter ?? '', ] .filter((f) => f !== '') .join(','); const args: string[] = ['-i', segment]; if (vf) args.push('-vf', vf); args.push( '-r', String(opts.fps ?? 24), '-c:v', 'libx264', '-preset', 'fast', '-crf', String(opts.crf ?? 20), '-c:a', 'aac', '-b:a', opts.audioBitrate ?? STANDARD_AUDIO_BITRATE, '-ar', String(opts.sampleRate ?? 44100), '-ac', String(opts.channels ?? 2), '-movflags', '+faststart', ); // Pad/trim the audio to exactly the video length so the prepped segment is // A/V-aligned: apad makes the audio effectively infinite, -shortest then ends // the OUTPUT at the (finite) video stream. Keeps a demuxer `-c copy` concat // drift-free. Default on; opt out for a known audioless segment. if (opts.padAudioToVideoDuration !== false) { args.push('-af', 'apad', '-shortest'); } // -t (when set) just before the output path → caps the output duration. args.push(...trimTailArgs(opts.clipMaxSeconds), outputPath); return args; } export interface BuildMusicMixOptions { /** Mix level for the music bed (0..1). Default {@link DEFAULT_MUSIC_VOLUME}. */ volume?: number; /** Tail fade-out seconds. Default {@link DEFAULT_MUSIC_FADE_OUT_SEC}. */ fadeOutSec?: number; /** * Total video duration in seconds — used to compute the fade-out start * (`total - fadeOut`) and the `-t` cap. The caller probes this via * `ffprobeDuration`. Defaults to 0 (fade starts at 0 / `-t 0`), only used by * the dry-run / pure-builder path where the duration is not yet known. */ totalDurationSec?: number; /** * Voice-forward mix (default false → byte-identical legacy). When true, the * video's own audio (`[0:a]`, i.e. the baked per-scene narration) is loudnorm'd * to a broadcast target, the music bed is sidechain-ducked under it, and the * final mix is limited — so narration is never buried beneath the bed. This is * the film-proven chain (loudnorm I=-15:TP=-1.5 + sidechaincompress + alimiter). * assemble enables it by default whenever per-scene narration was produced. */ voiceForward?: boolean; } /** Format a number the way the Python f-strings do (`:.2f` / `:.3f`). */ function fixed(value: number, digits: number): string { return value.toFixed(digits); } /** * Build the background-music mix ffmpeg args (PURE). Loops the music under the * narration at a low volume with a tail fade-out, then mixes via amix. * * Ported VERBATIM from assembly_utils.add_background_music (L408-435): * -i video -stream_loop -1 -i music * -filter_complex "[0:a]volume=1.0[v]; * [1:a]volume={vol},afade=t=out:st={fade_start}:d={fade_out}[m]; * [v][m]amix=inputs=2:duration=first:dropout_transition=600:normalize=0[a]" * -map 0:v -map [a] * -c:v copy -c:a aac -b:a 192k * -movflags +faststart * -t {total} output * * fade_start = max(0, total - fade_out). `dropout_transition=600` prevents early * audio cutoff on silent sections; `normalize=0` keeps the narration at full * level while the music stays at `volume`. */ export function buildMusicMixArgs( videoPath: string, musicPath: string, outputPath: string, opts: BuildMusicMixOptions = {}, ): string[] { const volume = opts.volume ?? DEFAULT_MUSIC_VOLUME; const fadeOut = opts.fadeOutSec ?? DEFAULT_MUSIC_FADE_OUT_SEC; const totalDur = opts.totalDurationSec ?? 0; const fadeStart = Math.max(0, totalDur - fadeOut); // Voice-forward (opt-in) loudnorms the video's own audio (the narration in // [0:a]), ducks the music under it via sidechaincompress, and limits the final // mix — the film-proven chain that keeps narration above the bed. Default // (voiceForward unset/false) is the VERBATIM legacy add_background_music graph. const filterComplex = opts.voiceForward ? `[0:a]loudnorm=I=-15:TP=-1.5:LRA=11,asplit=2[v0][vsc];` + `[1:a]volume=${volume},afade=t=out:st=${fixed(fadeStart, 2)}:d=${fadeOut}[m];` + `[m][vsc]sidechaincompress=threshold=0.05:ratio=8:attack=5:release=250[mduck];` + `[v0][mduck]amix=inputs=2:duration=first:dropout_transition=600:normalize=0,alimiter=limit=0.97[a]` : `[0:a]volume=1.0[v];` + `[1:a]volume=${volume},afade=t=out:st=${fixed(fadeStart, 2)}:d=${fadeOut}[m];` + `[v][m]amix=inputs=2:duration=first:dropout_transition=600:normalize=0[a]`; return [ '-i', videoPath, '-stream_loop', '-1', '-i', musicPath, '-filter_complex', filterComplex, '-map', '0:v', '-map', '[a]', '-c:v', 'copy', '-c:a', 'aac', '-b:a', STANDARD_AUDIO_BITRATE, '-movflags', '+faststart', '-t', fixed(totalDur, 3), outputPath, ]; } // --------------------------------------------------------------------------- // Multi-layer audio mix // --------------------------------------------------------------------------- /** * A single audio layer to mix under or alongside the video's native audio. */ export interface AudioLayer { /** Path to the audio track file. */ trackPath: string; /** * Semantic role of this layer. * - `'music'` — background music bed (low volume, looped). * - `'narration'` — voice-over narration track (full volume, no loop). * - `'dialogue'` — on-screen dialogue (full volume, no loop). * - `'sfx'` — sound-effect track (mixed alongside, NOT a ducking * sidechain signal; does not by itself enable ducking). */ role: 'music' | 'narration' | 'dialogue' | 'sfx'; /** Mix level (0..1). Defaults: music → {@link DEFAULT_MUSIC_VOLUME}, voice → 1.0. */ volume?: number; /** * Whether to loop this track to fill the video duration. * Default: `true` for `music` layers, `false` for `narration`/`dialogue`. */ loop?: boolean; } export interface BuildMultiLayerMixOptions { /** * When `true` AND at least one `narration`/`dialogue` layer is present, * music layers are sidechained under the voice via `sidechaincompress` * (threshold −12 dB / 0.25 linear, ratio 4:1, attack 5 ms, release 200 ms) * so the music ducks automatically whenever voice is active. * Falls back to static-volume amix when there are no voice layers. * Default: `false`. */ duckMusicUnderVoice?: boolean; /** * Global music-bed volume applied to every `music` layer that does not * set its own `volume`. Individual `layer.volume` takes precedence. * Default: {@link DEFAULT_MUSIC_VOLUME} (0.05). */ musicVolume?: number; /** Reserved for callers that need to override the ffmpeg binary path. */ ffmpegBin?: string; } /** * Build ffmpeg args that mix the video's own audio together with N additional * audio layers. **PURE** — returns only the arg-string array; never spawns * processes, reads files, or has side-effects. * * ## Input numbering * - Input 0 : the video file (carries video + its original audio). * - Inputs 1…N : the supplied `layers`, in order. * * ## Filter graph — static-volume path (default) * ``` * [1:a]volume=[l1]; [2:a]volume=[l2]; … * [0:a][l1][l2]…amix=inputs=N+1:duration=first:dropout_transition=600:normalize=0[a] * ``` * * ## Filter graph — ducking path (`duckMusicUnderVoice=true` + voice layer present) * ``` * per-layer volume chains as above … * [voice_labels…]amix=…[voice] (or direct passthrough for single voice) * [voice]asplit=M+1[vs0]…[vsM-1][vsfinal] * [l_music0][vs0]sidechaincompress=…[duck0]; [l_music1][vs1]sidechaincompress=…[duck1]; … * [0:a][duck0][duck1]…[vsfinal][sfx…]amix=inputs=M+2+S:…[a] * ``` * where M = number of music layers and S = number of sfx layers. * SFX layers are mixed in at their own volume but are NEVER used as a * sidechain signal and do NOT by themselves enable ducking (only * `narration`/`dialogue` layers count as voice for the `duckMusicUnderVoice` * gate). * * ## Output mapping * `-map 0:v -map [a] -c:v copy -c:a aac -b:a 192k -movflags +faststart ` */ export function buildMultiLayerMixArgs( videoPath: string, layers: AudioLayer[], outputPath: string, opts: BuildMultiLayerMixOptions = {}, ): string[] { if (layers.length === 0) { throw new Error('buildMultiLayerMixArgs: at least one layer is required'); } const doDuck = opts.duckMusicUnderVoice ?? false; const globalMusicVol = opts.musicVolume ?? DEFAULT_MUSIC_VOLUME; // ── Input args ──────────────────────────────────────────────────────────── // Video is always input 0. Each layer follows; music layers get -stream_loop. const inputArgs: string[] = ['-i', videoPath]; for (const layer of layers) { const shouldLoop = layer.loop ?? layer.role === 'music'; if (shouldLoop) inputArgs.push('-stream_loop', '-1'); inputArgs.push('-i', layer.trackPath); } // ── Per-layer volume chains ─────────────────────────────────────────────── // Build these first (index-stable); categorise into music vs voice labels. const filterParts: string[] = []; const allLayerLabels: string[] = []; // [l1], [l2], … (in layer order) const musicLayerLabels: string[] = []; // subset with role === 'music' const voiceLayerLabels: string[] = []; // subset with role narration|dialogue const sfxLayerLabels: string[] = []; // subset with role === 'sfx' (mixed, never a sidechain) for (let i = 0; i < layers.length; i += 1) { const layer = layers[i]; const inputIdx = i + 1; // 0 is video const label = `[l${inputIdx}]`; allLayerLabels.push(label); const vol = layer.volume !== undefined ? layer.volume : layer.role === 'music' ? globalMusicVol : 1.0; filterParts.push(`[${inputIdx}:a]volume=${vol}${label}`); if (layer.role === 'music') { musicLayerLabels.push(label); } else if (layer.role === 'sfx') { // SFX is mixed alongside everything else but must never serve as a // sidechain signal and must not by itself enable music ducking. sfxLayerLabels.push(label); } else { // narration | dialogue → genuine voice signal that drives ducking. voiceLayerLabels.push(label); } } const hasVoice = voiceLayerLabels.length > 0; const hasMusic = musicLayerLabels.length > 0; if (doDuck && hasVoice && hasMusic) { // ── Sidechain-ducking path ────────────────────────────────────────────── // // Step 1: Merge all voice labels into a single [voice] stream. // - One voice label → simple volume=1.0 passthrough (rename). // - Multiple labels → amix down to mono [voice]. const voiceLabel = '[voice]'; if (voiceLayerLabels.length === 1) { filterParts.push(`${voiceLayerLabels[0]}volume=1.0${voiceLabel}`); } else { const joined = voiceLayerLabels.join(''); filterParts.push( `${joined}amix=inputs=${voiceLayerLabels.length}:duration=first:dropout_transition=600:normalize=0${voiceLabel}`, ); } // Step 2: asplit [voice] into M+1 copies: // [vs0]…[vsM-1] → one per music layer (sidechain signal) // [vsfinal] → passed into the final amix so voice stays audible const numMusicLayers = musicLayerLabels.length; const totalSplits = numMusicLayers + 1; const splitOutputLabels = Array.from( { length: totalSplits }, (_, k) => (k < numMusicLayers ? `[vs${k}]` : '[vsfinal]'), ); filterParts.push(`${voiceLabel}asplit=${totalSplits}${splitOutputLabels.join('')}`); // Step 3: Sidechain-compress each music layer against its voice copy. // sidechaincompress: [main][sidechain] → [duckN] const duckLabels: string[] = []; for (let mi = 0; mi < numMusicLayers; mi += 1) { const duckLabel = `[duck${mi}]`; filterParts.push( `${musicLayerLabels[mi]}[vs${mi}]sidechaincompress=threshold=0.25:ratio=4:attack=5:release=200${duckLabel}`, ); duckLabels.push(duckLabel); } // Step 4: Final amix — video audio + all ducked music + the final voice // copy + any sfx layers (mixed at full/own volume, NOT ducked). // inputs = 1 (video) + M (ducked music) + 1 (vsfinal) + S (sfx) const finalMixInputs = ['[0:a]', ...duckLabels, '[vsfinal]', ...sfxLayerLabels].join(''); const finalMixCount = 1 + duckLabels.length + 1 + sfxLayerLabels.length; filterParts.push( `${finalMixInputs}amix=inputs=${finalMixCount}:duration=first:dropout_transition=600:normalize=0[a]`, ); } else { // ── Static-volume amix path (default / legacy-compatible) ─────────────── // Mix: [0:a] + [l1] + [l2] + … → amix → [a] const allInputs = ['[0:a]', ...allLayerLabels].join(''); const totalInputs = 1 + allLayerLabels.length; filterParts.push( `${allInputs}amix=inputs=${totalInputs}:duration=first:dropout_transition=600:normalize=0[a]`, ); } // ── Output args ─────────────────────────────────────────────────────────── return [ ...inputArgs, '-filter_complex', filterParts.join(';'), '-map', '0:v', '-map', '[a]', '-c:v', 'copy', '-c:a', 'aac', '-b:a', STANDARD_AUDIO_BITRATE, '-movflags', '+faststart', outputPath, ]; } /** * Compose the full ordered layer list for the multi-layer mix step: the * `music` bed (if present) first as a looped `music` layer, then the extra * `audioLayers` (dialogue/sfx/voice) in order. Pure. Only called when * `input.audioLayers` is non-empty, so the result always has ≥1 layer. */ function combinedAudioLayers(input: StitchInput): AudioLayer[] { const musicLayer: AudioLayer[] = input.music ? [ { trackPath: input.music.trackPath, role: 'music', loop: true, ...(input.music.volume !== undefined ? { volume: input.music.volume } : {}), }, ] : []; return [...musicLayer, ...(input.audioLayers ?? [])]; } export interface StitchOptions extends RunFfmpegOptions { /** Override the ffprobe binary (forwarded to `ffprobeDuration`). */ ffprobeBin?: string; /** * Skip the pre-concat MP4-validity guard. Default OFF (validation ON) for * real runs. The guard probes each input segment with {@link isValidMp4} and * fails fast on a truncated/no-moov MP4 instead of letting ffmpeg concat die * with a cryptic "Invalid data found when processing input". Always skipped on * dry-run (no real files to probe). */ skipSegmentValidation?: boolean; } /** * Orchestrate the stitch: write the concat list (demuxer path), pick the * demuxer-vs-filter path, run via `runFfmpeg`, optionally mix a music bed, and * return the final MP4 path + a plan of the executed command sequence. * * On `dryRun`, returns the planned command sequence WITHOUT writing the concat * list, spawning ffmpeg, or probing durations (music fade-start is computed * from 0). This is the path unit tests use. * * NOTE: the real-spawn path. The final-MP4 quality check is a HUMAN integration * checkpoint — out of scope here. */ export async function stitch( input: StitchInput, opts: StitchOptions = {}, ): Promise { const segs = orderedSegments(input); if (segs.length === 0) { // Defensive: nothing to concat. Caller is expected to pass >=1 segment. throw new Error('stitch: no segments to concatenate'); } const strategy = selectConcatStrategy(input.concatStrategy ?? 'auto', segs.length); const hasMusic = input.music !== undefined; // Extra (dialogue/sfx/voice) global layers beyond the music bed. Presence // here flips the mix step from the legacy single-music buildMusicMixArgs path // to the multi-layer buildMultiLayerMixArgs path. Empty/undefined ⇒ legacy. const hasExtraLayers = (input.audioLayers?.length ?? 0) > 0; // Any audio mix at all (music bed and/or extra layers) requires the // concat→intermediate→mix two-step. When neither is present, concat targets // the final output directly (byte-identical legacy no-audio path). const hasAudioMix = hasMusic || hasExtraLayers; const concatListPath = input.concatListPath ?? resolvePath(dirname(input.outputPath), 'concat.txt'); // When audio is mixed, concat writes an intermediate file and the mix step // produces the final output (mirrors nex_assemble's concat_no_music.mp4). const concatOutput = hasAudioMix ? resolvePath(dirname(input.outputPath), 'concat_no_music.mp4') : input.outputPath; const plan: StitchPlannedStep[] = []; // --- Optional per-segment normalization pre-pass (WS9) --- // When a tail-cut or letterbox is requested, each ordered segment is // re-encoded to the standard (dropping dead tail frames / adding cinematic // bars) before concat; the concat then consumes the normalized clips. Default // (neither field set) emits no prep steps and is byte-identical to legacy. const hasGrade = (typeof input.gradeLut === 'string' && input.gradeLut !== '') || (typeof input.gradeId === 'string' && input.gradeId !== '') || (input.segmentGradeIds?.some((g) => typeof g === 'string' && g !== '') ?? false); const needsPrep = (input.clipMaxSeconds !== undefined && input.clipMaxSeconds > 0) || (typeof input.letterboxRatio === 'string' && input.letterboxRatio !== '') || hasGrade; const prepDir = resolvePath(dirname(input.outputPath), '.prep'); const concatSegs = needsPrep ? segs.map((_, i) => resolvePath(prepDir, `seg-${String(i).padStart(3, '0')}.mp4`)) : segs; if (needsPrep) { for (let i = 0; i < segs.length; i += 1) { const seg = segs[i]; // Per-segment grade override realizes the narrative color language; falls // back to the uniform gradeId. const gradeFilter = resolveGradeFilter(input.segmentGradeIds?.[i] ?? input.gradeId); // The audio-pad (apad) keeps the prepped segment A/V-aligned so the demuxer // `-c copy` concat cannot drift — but apad needs an audio stream, and the // demuxer path (unlike the filter path) happily concats video-only segments // (a silent intro/outro sting, or a raw provider clip). Probe on real runs // and disable the pad when a segment has no audio; dry-run can't probe (and // never executes ffmpeg) so it keeps the default. let padAudioToVideoDuration = true; if (!opts.dryRun) { try { const probe = await probeMedia(seg, { ffprobeBin: opts.ffprobeBin }); padAudioToVideoDuration = probe.audioPresent; } catch { padAudioToVideoDuration = false; } } plan.push({ kind: 'segment-prep', args: buildSegmentPrepArgs(seg, concatSegs[i], { clipMaxSeconds: input.clipMaxSeconds, letterboxRatio: input.letterboxRatio, width: input.letterboxCanvas?.width, height: input.letterboxCanvas?.height, gradeLut: input.gradeLut, gradeFilter, padAudioToVideoDuration, }), outputPath: concatSegs[i], }); } } // --- Optional reading-holds (motion-comic) --- // For each designated segment index, generate a held readable still of its // first frame and INSERT it before the segment in the concat. Any hold forces // the FILTER concat (re-encode) so the freshly-encoded holds concat cleanly // with the (possibly differently-encoded) segments. Empty ⇒ unchanged. const holdIndices = new Set( (input.readingHold?.segments ?? []).filter((i) => Number.isInteger(i) && i >= 0 && i < segs.length), ); const holdDir = resolvePath(dirname(input.outputPath), '.holds'); let finalConcatSegs = concatSegs; let effectiveStrategy = strategy; if (holdIndices.size > 0) { const holdSec = input.readingHold?.holdSec ?? 3.5; const expanded: string[] = []; for (let i = 0; i < concatSegs.length; i += 1) { if (holdIndices.has(i)) { // The hold must match the size of the clip it sits beside, or the forced // FILTER concat rejects it (it does not rescale inputs). With a letterbox // prep the prepped segments are the canvas size; otherwise the // concatenated clip keeps its native size — so probe it (real runs) and // match. Dry-run can't probe → canvas / 1280×720 default (args unexecuted). let holdW = input.letterboxCanvas?.width ?? 1280; let holdH = input.letterboxCanvas?.height ?? 720; if (!opts.dryRun && !input.letterboxRatio) { try { const dim = await probeMedia(segs[i], { ffprobeBin: opts.ffprobeBin }); if (dim.width && dim.height) { holdW = dim.width; holdH = dim.height; } } catch { /* unprobeable → fall back to canvas / default */ } } const holdPath = resolvePath(holdDir, `hold-${String(i).padStart(3, '0')}.mp4`); plan.push({ kind: 'reading-hold', args: buildReadingHoldArgs(concatSegs[i], holdPath, { holdSec, width: holdW, height: holdH }), outputPath: holdPath, }); expanded.push(holdPath); } expanded.push(concatSegs[i]); } finalConcatSegs = expanded; effectiveStrategy = 'filter'; } // --- Concat step --- const concatArgs = effectiveStrategy === 'demuxer' ? buildConcatDemuxerArgs(finalConcatSegs, concatListPath, concatOutput) : buildConcatFilterArgs(finalConcatSegs, concatOutput); plan.push({ kind: effectiveStrategy === 'demuxer' ? 'concat-demuxer' : 'concat-filter', args: concatArgs, outputPath: concatOutput, }); // --- Audio-mix step (optional) --- // Two mutually-exclusive paths: // * hasExtraLayers → multi-layer mix (music bed prepended + dialogue/sfx). // * music-only → the EXACT legacy buildMusicMixArgs path (byte-identical). // Neither ⇒ no mix step (legacy no-audio path). if (hasExtraLayers) { const layers = combinedAudioLayers(input); const mixArgs = buildMultiLayerMixArgs(concatOutput, layers, input.outputPath, { ...(input.duckMusicUnderVoice ? { duckMusicUnderVoice: true } : {}), }); plan.push({ kind: 'multi-layer-mix', args: mixArgs, outputPath: input.outputPath }); } else if (hasMusic && input.music) { // The fade-start / -t cap need the concat duration; on dry-run we leave it 0. const musicArgs = buildMusicMixArgs(concatOutput, input.music.trackPath, input.outputPath, { volume: input.music.volume, fadeOutSec: input.music.fadeOutSec, totalDurationSec: 0, voiceForward: input.music.voiceForward, }); plan.push({ kind: 'music-mix', args: musicArgs, outputPath: input.outputPath }); } if (opts.dryRun) { return { status: 'dry-run', outputPath: input.outputPath, orderedSegments: segs, concatStrategy: effectiveStrategy, music: hasMusic, plan, durationMs: 0, }; } // --- Pre-concat corruption guard --- // useapi sometimes reports a generation 'complete' while the downloaded mp4 // is truncated (no moov atom): it passes existence/size checks but ffmpeg // concat later dies with "Invalid data found when processing input". Probe // each real input segment up front so the corrupt one is named clearly. // Only the actual segment inputs are probed (not derived intermediates). if (!opts.skipSegmentValidation) { for (const seg of segs) { const ok = await isValidMp4(seg, { ffprobeBin: opts.ffprobeBin }); if (!ok) { throw new VclawError( 'ffmpeg_failed', `stitch input segment is a corrupt or truncated MP4 (ffprobe could not read a valid duration): ${seg}`, { segment: seg }, ); } } } // --- Real execution --- await mkdir(dirname(input.outputPath), { recursive: true }); // Per-segment normalization pre-pass: concat then reads the normalized clips. if (needsPrep) { await mkdir(prepDir, { recursive: true }); for (const step of plan) { if (step.kind === 'segment-prep') await runFfmpeg(step.args, opts); } } // Reading-hold clips (motion-comic): built from each designated segment's first // frame AFTER any prep pass, then inserted before it in the concat. if (holdIndices.size > 0) { await mkdir(holdDir, { recursive: true }); for (const step of plan) { if (step.kind === 'reading-hold') await runFfmpeg(step.args, opts); } } if (effectiveStrategy === 'demuxer') { await mkdir(dirname(concatListPath), { recursive: true }); await writeFile(concatListPath, buildConcatListContent(finalConcatSegs), 'utf8'); } await runFfmpeg(concatArgs, opts); let finalPath = concatOutput; if (hasExtraLayers) { // Multi-layer mix: dialogue/sfx (+ optional music bed) over the concat. // buildMultiLayerMixArgs uses duration=first (driven by the video input), // so it needs no probed total — the args are already final from planning. const mixStep = plan.find((s) => s.kind === 'multi-layer-mix'); const mixArgs = mixStep?.args ?? buildMultiLayerMixArgs(concatOutput, combinedAudioLayers(input), input.outputPath); await runFfmpeg(mixArgs, opts); finalPath = input.outputPath; } else if (hasMusic && input.music) { // Probe the concat output to compute the real fade-start + -t cap. const concatMs = await ffprobeDuration(concatOutput, { ffprobeBin: opts.ffprobeBin }); const musicArgs = buildMusicMixArgs(concatOutput, input.music.trackPath, input.outputPath, { volume: input.music.volume, fadeOutSec: input.music.fadeOutSec, totalDurationSec: concatMs / 1000, voiceForward: input.music.voiceForward, }); // Refresh the plan's music step with the duration-resolved args. const musicStep = plan.find((s) => s.kind === 'music-mix'); if (musicStep) musicStep.args = musicArgs; await runFfmpeg(musicArgs, opts); finalPath = input.outputPath; } const durationMs = await ffprobeDuration(finalPath, { ffprobeBin: opts.ffprobeBin }); // Touch stat so a 0-byte output surfaces as a runtime failure path-side. await stat(finalPath); return { status: 'complete', outputPath: finalPath, orderedSegments: segs, concatStrategy: strategy, music: hasMusic, plan, durationMs, }; }