/** * Native local overlay renderer (no provider spend, no moderation). * * Renders each transcript segment as a broadcast lower-third — an SVG rasterised * to PNG via `sharp` (resvg text shaping; NO system freetype / ffmpeg `drawtext` * needed) — then composites the cards onto the source clip with a single ffmpeg * `overlay` filtergraph (each card fades + slides up on its segment window). The * original audio is preserved. * * This is the reliable production path for motion-graphics overlays: the Google * Flow omni-flash V2V "add-overlay" edit is moderation-blocked for most inputs * (FINISH_REASON_INPUT_VIDEO_EDIT), whereas this runs fully locally. * * The pure pieces (SVG builder, filtergraph builder, accent-word splitter) are * unit-tested; the sharp rasterise + the ffmpeg run are injectable side effects. */ import sharp from 'sharp'; import { mkdir, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { iconMotifSvg, statMotifSvg, type MotifMoment } from './motifs.js'; /** One caption: the spoken text and its absolute window in the source clip. */ export interface LocalCaption { text: string; start: number; end: number; } /** * The three card looks, ported from the source skill's style catalog so * `--render local --style X` actually changes the rendered overlay (previously * only the V2V *prompt* varied by style; the local card was fixed). * - `apple-clean` — rounded dark frosted glass, clean sans-serif sentence case. * - `editorial-dark` — squared solid-black poster panel, bold condensed UPPERCASE. * - `knowledge-tool` — soft cool-dark card, editorial serif, calm lavender accent. */ export type CardVariant = 'apple-clean' | 'editorial-dark' | 'knowledge-tool'; interface VariantSpec { /** Panel geometry: floating inset card · full-bleed bar · left-anchored card. */ layout: 'inset' | 'fullbleed' | 'left'; panelFill: string; panelRx: number; stroke: string; fontFamily: string; /** Font for the kicker/label row (mono for the knowledge `[[ ]]` look). */ labelFamily: string; fontWeight: number; uppercase: boolean; letterSpacing: number; textFill: string; /** Headline (giant anchor word) font family for this variant. */ headlineFamily: string; /** Accent treatment: a vertical bar at the panel's left, or a full-width top rule. */ accentRule: 'left-bar' | 'top-rule'; accentRuleSize: number; /** Kicker style: accent dot · solid accent tag block · `[[ MONO ]]` brackets. */ label: 'dot' | 'tag' | 'brackets'; } // Three structurally distinct looks faithful to the source skill's catalog: // - apple-clean — a floating inset rounded frosted card, thin left accent bar, // accent dot + uppercase kicker, sans-serif sentence case. // - editorial-dark — a FULL-BLEED edge-to-edge near-black poster bar, thick accent // TOP rule, a solid accent tag block label, bold condensed CAPS. // - knowledge-tool — a LEFT-anchored narrower cool-dark study card, thick lavender // left rule, a `[[ MONO ]]` bracket label, editorial serif body. const VARIANTS: Record = { 'apple-clean': { layout: 'inset', panelFill: 'rgba(14,14,18,0.86)', panelRx: 22, stroke: 'rgba(255,255,255,0.15)', fontFamily: 'Helvetica Neue, Helvetica, Arial, sans-serif', labelFamily: 'Helvetica Neue, Helvetica, Arial, sans-serif', fontWeight: 600, uppercase: false, letterSpacing: 0, textFill: '#ffffff', headlineFamily: 'Helvetica Neue, Helvetica, Arial, sans-serif', accentRule: 'left-bar', accentRuleSize: 8, label: 'dot', }, 'editorial-dark': { layout: 'fullbleed', panelFill: 'rgba(0,0,0,0.94)', panelRx: 0, stroke: '', fontFamily: 'Arial Narrow, Helvetica Neue, Arial, sans-serif', labelFamily: 'Arial Narrow, Helvetica, Arial, sans-serif', fontWeight: 800, uppercase: true, letterSpacing: 1, textFill: '#ffffff', headlineFamily: 'Arial Narrow, Helvetica Neue, Arial, sans-serif', accentRule: 'top-rule', accentRuleSize: 10, label: 'tag', }, 'knowledge-tool': { layout: 'left', panelFill: 'rgba(19,19,24,0.93)', panelRx: 12, stroke: 'rgba(167,139,250,0.28)', fontFamily: 'Georgia, Times New Roman, serif', labelFamily: 'Menlo, Consolas, monospace', fontWeight: 500, uppercase: false, letterSpacing: 0, textFill: '#e9e6f4', headlineFamily: 'Georgia, Times New Roman, serif', accentRule: 'left-bar', accentRuleSize: 12, label: 'brackets', }, }; /** Resolve a variant id (defaults to apple-clean — the original card look). */ function variantSpec(v: CardVariant | undefined): VariantSpec { return VARIANTS[v ?? 'apple-clean']; } /** Visual config for the lower-third (accent hex + small kicker label + style). */ export interface LocalRenderStyle { accent: string; /** Optional small kicker label above the caption (e.g. a brand). Omitted when blank. */ kicker?: string; /** Lowercased words that get the accent colour (e.g. brand/anchor words). */ accentWords?: string[]; /** Which card look to render (default `apple-clean`). */ variant?: CardVariant; /** * Word-by-word reveal: only the first N words of the caption are visible; the rest * render transparent (so the panel keeps its full size — no per-frame reflow). The * animated renderer drives this with `revealWordCount(t)`. Omitted → all words. */ visibleWords?: number; } /** Escape the five XML entities so arbitrary transcript text is SVG-safe. Pure. */ export function xmlEscape(text: string): string { return text .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"') .replace(/'/g, '''); } /** * Word-wrap `text` to at most `maxChars` per line (greedy by words). Pure — used * to size the panel and lay out tspans. */ export function wrapWords(text: string, maxChars: number): string[] { const words = text.trim().split(/\s+/).filter(Boolean); const lines: string[] = []; let line = ''; for (const w of words) { if (line && line.length + 1 + w.length > maxChars) { lines.push(line); line = w; } else { line = line ? `${line} ${w}` : w; } } if (line) lines.push(line); return lines.length ? lines : ['']; } /** * Render a wrapped line as ``s. Accent words render in the accent colour AND * (per the source skill) get a small accent **asterisk** beat-marker after them. * Pure. */ function lineTspans( line: string, accent: string, baseFill: string, accentWords: Set, reveal?: { counter: { n: number }; visible: number }, ): string { return line .split(/(\s+)/) .map((tok) => { if (/^\s+$/.test(tok)) return xmlEscape(tok); // Word-by-word reveal: words past the visible count render transparent (they // still occupy space so the panel does not reflow frame to frame). let hidden = false; if (reveal) { reveal.counter.n += 1; hidden = reveal.counter.n > reveal.visible; } // fill-opacity (not opacity) — resvg/sharp reliably honours it on . const op = hidden ? ' fill-opacity="0"' : ''; const bare = tok.toLowerCase().replace(/[^a-z0-9]/g, ''); if (accentWords.has(bare)) { return `${xmlEscape(tok)}*`; } return `${xmlEscape(tok)}`; }) .join(''); } /** * Build the lower-third SVG for one caption. Pure & deterministic. `style.variant` * selects a **structurally distinct** card per the source skill's catalog — a * floating inset frosted card (apple-clean), a full-bleed near-black poster bar with * an accent top rule and a solid accent tag (editorial-dark), or a left-anchored cool * study card with a `[[ MONO ]]` bracket label and serif body (knowledge-tool). */ export function lowerThirdSvg( caption: Pick, dims: { width: number; height: number }, style: LocalRenderStyle, ): string { const { width: W, height: H } = dims; const spec = variantSpec(style.variant); const accent = style.accent; const accentWords = new Set((style.accentWords ?? []).map((w) => w.toLowerCase().replace(/[^a-z0-9]/g, ''))); // Layout geometry differs per variant — this is the main structural distinction. const px = spec.layout === 'fullbleed' ? 0 : 36; const panelW = spec.layout === 'fullbleed' ? W : spec.layout === 'left' ? Math.round(W * 0.62) : W - 72; const padX = 44; const tx = px + padX; const topPad = spec.accentRule === 'top-rule' ? spec.accentRuleSize : 0; // Smaller, less shouty lower-third text (a bit tighter than the original 46px) — // the smaller font also fits more characters per line, so captions wrap less. const fontSize = 38; const maxChars = Math.max(12, Math.floor((panelW - padX - 56) / 20)); const raw = caption.text.replace(/\s+$/, ''); const text = spec.uppercase ? raw.toUpperCase() : raw; const lines = wrapWords(text, maxChars); const lineH = fontSize + 14; const labelH = 42; const panelH = topPad + 30 + labelH + lines.length * lineH + 22; const py = H - panelH - 70; const labelBaseline = py + topPad + 40; const textTop = labelBaseline + 50; const lsAttr = spec.letterSpacing ? ` letter-spacing="${spec.letterSpacing}"` : ''; // Word-by-word reveal: a shared counter walks every word across all lines. const reveal = style.visibleWords !== undefined ? { counter: { n: 0 }, visible: style.visibleWords } : undefined; const tspanLines = lines .map((l, i) => `${lineTspans(l, accent, spec.textFill, accentWords, reveal)}`) .join(''); const accentRuleSvg = spec.accentRule === 'top-rule' ? `` : ``; const kicker = (style.kicker ?? '').trim().toUpperCase(); let labelSvg = ''; if (kicker) { if (spec.label === 'tag') { const tagW = 24 + kicker.length * 14; labelSvg = ` ${xmlEscape(kicker)}`; } else if (spec.label === 'brackets') { labelSvg = `${xmlEscape(`[[ ${kicker} ]]`)}`; } else { labelSvg = ` ${xmlEscape(kicker)}`; } } const strokeSvg = spec.stroke ? `` : ''; return ` ${strokeSvg} ${accentRuleSvg} ${labelSvg} ${tspanLines} `; } /** * Build the giant **anchor-headline** SVG — a single dominant keyword centred in * the upper-middle of the frame, with the accent asterisk beat-marker after it * (the source skill's signature kinetic-typography moment). Pure & deterministic. */ export function anchorHeadlineSvg( word: string, dims: { width: number; height: number }, style: LocalRenderStyle, ): string { const { width: W, height: H } = dims; const spec = variantSpec(style.variant); const raw = word.trim(); const text = spec.uppercase || (style.variant ?? 'apple-clean') !== 'knowledge-tool' ? raw.toUpperCase() : raw; // Size to fit the frame width (rough advance ~0.6em per glyph), capped. const fitted = Math.floor((W * 0.92) / Math.max(1, text.length) / 0.6); const fontSize = Math.max(64, Math.min(Math.floor(W * 0.2), fitted)); const cy = Math.floor(H * 0.42); return ` ${xmlEscape(text)}* `; } /** Rasterise a lower-third SVG to a PNG buffer via sharp. Side effect (sharp). */ export async function rasteriseCaption(svg: string): Promise { return sharp(Buffer.from(svg)).png().toBuffer(); } /** * Build the ffmpeg `-filter_complex` graph compositing N pre-rendered full-frame * card PNGs (inputs 1..N; input 0 is the source video) onto the source — each * card fades in/out and slides up 36px on entrance over its [start,end] window. * Pure (string only). */ export function buildLocalFiltergraph(cards: Array<{ start: number; end: number }>): string { if (cards.length === 0) return '[0:v]null[vout]'; const parts: string[] = []; let last = '[0:v]'; cards.forEach((c, i) => { const start = c.start; const end = Math.max(c.end, start + 1.0); parts.push( `[${i + 1}:v]format=rgba,fade=t=in:st=${start.toFixed(2)}:d=0.30:alpha=1,` + `fade=t=out:st=${(end - 0.3).toFixed(2)}:d=0.30:alpha=1[c${i}]`, ); const out = i === cards.length - 1 ? '[vout]' : `[v${i}]`; const yExpr = `'if(lt(t,${(start + 0.35).toFixed(2)}), 36*(1-(t-${start.toFixed(2)})/0.35), 0)'`; parts.push(`${last}[c${i}]overlay=x=0:y=${yExpr}:enable='between(t,${start.toFixed(2)},${end.toFixed(2)})'${out}`); last = `[v${i}]`; }); return parts.join(';'); } /** * Build the ffmpeg argv for the overlay composite. Pure — the source is input 0, * each card PNG is a looped image input, the graph maps [vout] + the source audio. */ export function buildLocalRenderArgs( sourceVideo: string, cardPngPaths: string[], filtergraph: string, outputPath: string, ): string[] { const args = ['-y', '-i', sourceVideo]; for (const p of cardPngPaths) args.push('-loop', '1', '-i', p); args.push( '-filter_complex', filtergraph, '-map', '[vout]', '-map', '0:a?', '-c:v', 'libx264', '-crf', '18', '-preset', 'fast', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-b:a', '192k', '-movflags', '+faststart', '-shortest', outputPath, ); return args; } /** Runs ffmpeg with the given argv. Injectable so the orchestrator tests offline. */ export type FfmpegRunner = (args: string[]) => Promise; export interface RenderLocalDeps { ffmpeg: FfmpegRunner; /** Override the rasteriser (tests). Defaults to the live sharp path. */ rasterise?: (svg: string) => Promise; } /** A giant anchor-headline moment: the word + the short window it pulses on. */ export interface HeadlineMoment { word: string; start: number; end: number; } export interface RenderLocalOptions { sourceVideo: string; captions: LocalCaption[]; dims: { width: number; height: number }; style: LocalRenderStyle; /** Optional giant anchor-headline overlays, composited above the lower-thirds. */ headlines?: HeadlineMoment[]; /** Optional concept icons / stat callouts, composited in the upper third. */ motifs?: MotifMoment[]; /** Dir for the intermediate card PNGs. */ cardDir: string; outputPath: string; } /** Resolve a motif into its full-frame SVG using the active style. Pure. */ export function motifSvg(m: MotifMoment, dims: { width: number; height: number }, style: LocalRenderStyle): string { const spec = variantSpec(style.variant); return m.kind === 'stat' ? statMotifSvg(m, dims, { accent: style.accent, numberFamily: spec.headlineFamily }) : iconMotifSvg(m, dims, { accent: style.accent, labelFamily: spec.labelFamily }); } /** * Render the local overlay reel: rasterise one lower-third per caption (plus any * giant anchor-headline moments), then composite all elements onto the source with * the fade/slide filtergraph, sorted by start time. Returns the list of card PNG * paths written (for provenance / tests). */ export async function renderLocalOverlay(opts: RenderLocalOptions, deps: RenderLocalDeps): Promise { const rasterise = deps.rasterise ?? rasteriseCaption; await mkdir(opts.cardDir, { recursive: true }); await mkdir(dirname(opts.outputPath), { recursive: true }); // Build a unified, start-sorted element list: lower-thirds + headline pulses + // concept icon / stat motifs (the "visualize what's being said" layer). const elements: Array<{ svg: string; start: number; end: number }> = [ ...opts.captions.map((c) => ({ svg: lowerThirdSvg(c, opts.dims, opts.style), start: c.start, end: c.end })), ...(opts.headlines ?? []).map((h) => ({ svg: anchorHeadlineSvg(h.word, opts.dims, opts.style), start: h.start, end: h.end })), ...(opts.motifs ?? []).map((m) => ({ svg: motifSvg(m, opts.dims, opts.style), start: m.start, end: m.end })), ].sort((a, b) => a.start - b.start); const cardPaths: string[] = []; for (let i = 0; i < elements.length; i++) { const png = await rasterise(elements[i].svg); const p = join(opts.cardDir, `card-${String(i).padStart(2, '0')}.png`); await writeFile(p, png); cardPaths.push(p); } const filtergraph = buildLocalFiltergraph(elements.map((e) => ({ start: e.start, end: e.end }))); await deps.ffmpeg(buildLocalRenderArgs(opts.sourceVideo, cardPaths, filtergraph, opts.outputPath)); return cardPaths; }