/** * Public TTS entrypoint for the assemble stage (sub-slice 3b). * * Source of truth: `skills/video-replicator/scripts/generate_tts.py`. * * Wires the text->speech path through the ElevenLabs adapter, with * transcript-file loading (`transcript.ts` — standalone + SEALCAM+-embedded * shapes, editable per-scene overrides) and the optional combined narration * track (`audio-concat.ts` — concat with per-scene silence padding to slide * durations). The remaining Python behaviors are deferred to later 3b * sub-commits and explicitly marked TODO below. */ import { writeFile, mkdir } from 'node:fs/promises'; import { join } from 'node:path'; import { synthesizeSpeech, DEFAULT_MODEL_ID, DEFAULT_STABILITY, DEFAULT_SIMILARITY_BOOST, type VoiceSettings, } from './tts-elevenlabs.js'; import { validateOutputFormat, DEFAULT_OUTPUT_FORMAT, sceneAudioFilename, type OutputFormat, } from './audio-utils.js'; import { ffprobeDuration } from './ffmpeg.js'; import { loadTranscriptFile, loadEditedTranscriptFile, transcriptToTtsSegments } from './transcript.js'; import { planNarrationConcat, runNarrationConcat } from './audio-concat.js'; import type { AssembleManifestEntry } from './types.js'; /** Filename of the combined narration track (always mp3, see audio-concat.ts). */ export const COMBINED_NARRATION_FILENAME = 'narration-combined.mp3'; /** A single narration segment: one scene's text. */ export interface TtsSegment { /** 1-based scene index, used for filenames + manifest ordering. */ sceneIndex: number; text: string; } export interface TtsInput { /** * Path to a transcript JSON file (per-scene narration). Standalone and * SEALCAM+-embedded shapes are supported — see `transcript.ts`. */ transcriptPath?: string; /** Optional editable-transcript per-scene overrides (editable_transcript.json). */ transcriptEditsPath?: string; /** Inline segments, used instead of `transcriptPath` when provided. */ segments?: TtsSegment[]; voiceId: string; modelId?: string; stability?: number; similarityBoost?: number; style?: number; speed?: number; /** Boost similarity to original voice (default true). */ speakerBoost?: boolean; outputFormat?: string; /** Directory to write per-scene + combined audio into. */ outputDir: string; /** Read from ELEVENLABS_API_KEY when omitted. */ apiKey?: string; /** Skip provider calls and file writes; report intended actions only. */ dryRun?: boolean; /** Optional override of ffprobe path for duration probing. */ ffprobeBin?: string; /** * Concatenate the per-scene audio into a single narration track * ({@link COMBINED_NARRATION_FILENAME} in `outputDir`). Default off. */ combine?: boolean; /** * With `combine`: pad each scene's audio with trailing silence to its scene * duration so the combined track stays aligned to the slide timeline. */ padToDuration?: boolean; /** Target per-scene on-screen durations, seconds (in scene order). */ sceneDurationsSec?: number[]; /** Optional override of ffmpeg path for the combined-track concat. */ ffmpegBin?: string; } export interface TtsSceneOutput { sceneIndex: number; path: string; durationMs: number; sizeBytes: number; } export interface TtsResult { status: 'complete' | 'dry-run'; /** Per-scene audio file paths. */ scenes: TtsSceneOutput[]; /** Path to the combined narration track (set when `combine` is on). */ combinedPath?: string; manifest: AssembleManifestEntry[]; /** Advisory notes (e.g. multi-voice scenes flattened to a single voice). */ warnings: string[]; } /** * Generate per-scene narration audio from a transcript or inline segments. * * TODO(3b later sub-commits): * - conductor / sync-to-slides: re-time slides to narration (the inverse of * `padToDuration`; `narration-fit.ts` plans this but the baker is unwired). * - bake-narration / speech-to-speech (swap) subcommands. * - per-speaker multi-voice synthesis (multi-voice scenes currently flatten * to one voice — see `transcript.ts`). */ export async function generateTts(input: TtsInput): Promise { const outputFormat: OutputFormat = validateOutputFormat( input.outputFormat ?? DEFAULT_OUTPUT_FORMAT, ); const { segments, warnings } = await resolveSegments(input); const combinedPath = input.combine ? join(input.outputDir, COMBINED_NARRATION_FILENAME) : undefined; const voiceSettings: VoiceSettings = { stability: input.stability ?? DEFAULT_STABILITY, similarity_boost: input.similarityBoost ?? DEFAULT_SIMILARITY_BOOST, style: input.style ?? 0.0, use_speaker_boost: input.speakerBoost ?? true, }; const modelId = input.modelId ?? DEFAULT_MODEL_ID; if (input.dryRun) { return { status: 'dry-run', scenes: segments.map((s) => ({ sceneIndex: s.sceneIndex, path: join(input.outputDir, sceneAudioFilename(s.sceneIndex, outputFormat)), durationMs: 0, sizeBytes: 0, })), ...(combinedPath ? { combinedPath } : {}), manifest: [], warnings, }; } await mkdir(input.outputDir, { recursive: true }); const scenes: TtsSceneOutput[] = []; const manifest: AssembleManifestEntry[] = []; for (const segment of segments) { const bytes = await synthesizeSpeech({ voiceId: input.voiceId, text: segment.text, modelId, voiceSettings, outputFormat, speed: input.speed, apiKey: input.apiKey, }); const filename = sceneAudioFilename(segment.sceneIndex, outputFormat); const path = join(input.outputDir, filename); await writeFile(path, bytes); const durationMs = await ffprobeDuration(path, { ffprobeBin: input.ffprobeBin }); scenes.push({ sceneIndex: segment.sceneIndex, path, durationMs, sizeBytes: bytes.byteLength }); manifest.push({ kind: 'narration', path, durationMs, sceneIndex: segment.sceneIndex, sizeBytes: bytes.byteLength, generator: `elevenlabs:${modelId}`, }); } if (combinedPath) { const plan = planNarrationConcat({ files: scenes.map((s) => s.path), output: combinedPath, tempDir: join(input.outputDir, 'concat-tmp'), padToDuration: input.padToDuration, actualDurationsSec: scenes.map((s) => s.durationMs / 1000), sceneDurationsSec: input.sceneDurationsSec, }); await runNarrationConcat(plan, { ffmpegBin: input.ffmpegBin }); const combinedDurationMs = await ffprobeDuration(combinedPath, { ffprobeBin: input.ffprobeBin }); manifest.push({ kind: 'narration', path: combinedPath, durationMs: combinedDurationMs, sizeBytes: 0, generator: 'ffmpeg:concat', }); } return { status: 'complete', scenes, ...(combinedPath ? { combinedPath } : {}), manifest, warnings, }; } async function resolveSegments( input: TtsInput, ): Promise<{ segments: TtsSegment[]; warnings: string[] }> { if (input.segments && input.segments.length > 0) { return { segments: input.segments, warnings: [] }; } if (input.transcriptPath) { const transcript = await loadTranscriptFile(input.transcriptPath); const editsResult = input.transcriptEditsPath ? await loadEditedTranscriptFile(input.transcriptEditsPath) : { edits: undefined, warnings: [] as string[] }; const resolved = transcriptToTtsSegments(transcript, { edits: editsResult.edits }); const warnings = [...editsResult.warnings, ...resolved.warnings]; if (resolved.segments.length === 0) { throw new Error( `generateTts: transcript ${input.transcriptPath} contains no scenes with speech.`, ); } return { segments: resolved.segments, warnings }; } throw new Error('generateTts: provide either `segments` or `transcriptPath`.'); }