/** * SCAFFOLD: Gemini TTS request/response per docs; verify against a real key. * * Gemini TTS backend — wraps the generativelanguage.googleapis.com * `gemini-2.5-flash-preview-tts:generateContent` endpoint. This is an * API-KEY product (NOT Vertex), so it resolves a key from the existing * Gemini key pool (GEMINI_API_KEYS / GOOGLE_API_KEYS / GOOGLE_API_KEY) via * `fetchGeminiWithPool`. When a caller passes `input.env` carrying a key the * process.env-backed pool cannot see, that key is forwarded as the pool's * `keyOverride` so library callers authenticate without mutating process.env. * * The model returns RAW PCM (signed 16-bit little-endian, 24kHz, mono) as * base64 in candidates[0].content.parts[].inlineData.data. We wrap that PCM * in a 44-byte WAV header and write the result to `outputPath` (a .wav). * * Duration is computed deterministically from the PCM byte count — no ffprobe: * durationMs = round(pcmBytes / (24000 * 2) * 1000) * * Availability: a Gemini key is resolvable (pool size > 0 or one of the env * vars is set). The backend is inert (unavailable) without a key and never * throws on the default path; on a missing/empty audio response it throws * VclawError('tts_failed', ...). */ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; import { fetchGeminiWithPool } from '../gemini-key-pool.js'; import { VclawError } from '../errors.js'; import type { TtsBackend, TtsGenInput, TtsGenResult } from './types.js'; const TTS_MODEL = 'gemini-2.5-flash-preview-tts'; const TTS_SAMPLE_RATE_HZ = 24_000; const TTS_BYTES_PER_SAMPLE = 2; // s16le const TTS_CHANNELS = 1; // mono const DEFAULT_VOICE = 'Kore'; /** Chars-per-second heuristic for dry-run duration estimation (~14 cps speech). */ const DRYRUN_CHARS_PER_SEC = 14; /** * Gemini API key sources, in the same precedence as the global key pool * (gemini-key-pool.ts). Used to resolve an explicit key from a caller-supplied * `input.env` so the pool (which only reads process.env) can be bypassed. */ const GEMINI_KEY_VARS = ['GEMINI_API_KEYS', 'GOOGLE_API_KEYS', 'GOOGLE_API_KEY'] as const; /** * Resolve the first key from a caller-supplied env that differs from * process.env. Returns undefined when no env was passed, or when the passed env * adds no key beyond what process.env already exposes (in which case the global * pool handles key selection). Splits on the same delimiters the pool uses and * returns the first token. */ function resolveKeyOverrideFromEnv(env: NodeJS.ProcessEnv | undefined): string | undefined { if (!env || env === process.env) return undefined; for (const varName of GEMINI_KEY_VARS) { const raw = env[varName]; if (typeof raw !== 'string' || raw.trim() === '') continue; // process.env already carries this exact value → let the pool handle it. if (process.env[varName] === raw) continue; const first = raw.split(/[,;\n\s]+/).map((t) => t.trim()).find((t) => t.length > 0); if (first) return first; } return undefined; } function buildTtsUrl(key: string): string { return `https://generativelanguage.googleapis.com/v1beta/models/${TTS_MODEL}:generateContent?key=${encodeURIComponent(key)}`; } /** * durationMs from raw PCM byte count (s16le 24kHz mono). * pcmBytes / (sampleRate * bytesPerSample * channels) * 1000 */ function pcmDurationMs(pcmBytes: number): number { return Math.round((pcmBytes / (TTS_SAMPLE_RATE_HZ * TTS_BYTES_PER_SAMPLE * TTS_CHANNELS)) * 1000); } /** Wrap raw PCM (s16le 24kHz mono) bytes in a canonical 44-byte WAV header. */ function wrapPcmInWav(pcm: Buffer): Buffer { const byteRate = TTS_SAMPLE_RATE_HZ * TTS_CHANNELS * TTS_BYTES_PER_SAMPLE; const blockAlign = TTS_CHANNELS * TTS_BYTES_PER_SAMPLE; const header = Buffer.alloc(44); header.write('RIFF', 0, 'ascii'); header.writeUInt32LE(36 + pcm.length, 4); // ChunkSize header.write('WAVE', 8, 'ascii'); header.write('fmt ', 12, 'ascii'); header.writeUInt32LE(16, 16); // Subchunk1Size (PCM) header.writeUInt16LE(1, 20); // AudioFormat = PCM header.writeUInt16LE(TTS_CHANNELS, 22); header.writeUInt32LE(TTS_SAMPLE_RATE_HZ, 24); header.writeUInt32LE(byteRate, 28); header.writeUInt16LE(blockAlign, 32); header.writeUInt16LE(TTS_BYTES_PER_SAMPLE * 8, 34); // BitsPerSample header.write('data', 36, 'ascii'); header.writeUInt32LE(pcm.length, 40); // Subchunk2Size return Buffer.concat([header, pcm]); } /** Defensive extraction of the first base64 inlineData audio payload. */ function extractAudioBase64(parsed: unknown): string | undefined { if (typeof parsed !== 'object' || parsed === null) return undefined; const candidates = (parsed as { candidates?: unknown }).candidates; if (!Array.isArray(candidates) || candidates.length === 0) return undefined; const parts = (candidates[0] as { content?: { parts?: unknown } })?.content?.parts; if (!Array.isArray(parts)) return undefined; for (const part of parts) { const data = (part as { inlineData?: { data?: unknown } })?.inlineData?.data; if (typeof data === 'string' && data.length > 0) return data; } return undefined; } export const geminiTts: TtsBackend = { id: 'gemini-tts', kind: 'tts', outputExtension: 'wav', displayName: 'Gemini TTS (gemini-2.5-flash-preview-tts)', // API-key product, not Vertex. Availability is resolved from the Gemini key // pool (see isTtsBackendAvailable), so requiredEnv stays empty and the // registry gates on requiresGeminiKey instead of this backend's id. requiredEnv: [], requiresVertex: false, requiresGeminiKey: true, summary: 'Single-speaker text-to-speech via the Gemini API (gemini-2.5-flash-preview-tts). Returns 24kHz mono PCM wrapped as WAV. Requires a Gemini API key (GEMINI_API_KEYS / GOOGLE_API_KEYS / GOOGLE_API_KEY).', async generate(input: TtsGenInput): Promise { const voice = input.voice && input.voice.trim() ? input.voice.trim() : DEFAULT_VOICE; if (!input.text || !input.text.trim()) { throw new VclawError('tts_failed', 'Cannot synthesize narration from empty text.', { backendId: 'gemini-tts', }); } // Dry-run: estimate duration from text length; do NOT touch network or do // real synthesis. Write a placeholder WAV (44-byte header, no samples) so // downstream existsSync() checks stay consistent. if (input.dryRun) { const estSec = Math.max(1, input.text.trim().length / DRYRUN_CHARS_PER_SEC); const durationMs = Math.round(estSec * 1000); await mkdir(dirname(input.outputPath), { recursive: true }); await writeFile(input.outputPath, wrapPcmInWav(Buffer.alloc(0))); return { path: input.outputPath, durationMs, backendId: 'gemini-tts' }; } const requestBody = JSON.stringify({ contents: [{ parts: [{ text: input.text }] }], generationConfig: { responseModalities: ['AUDIO'], speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voice }, }, }, }, }); // Resolve a key from input.env when it carries one the global pool (which // only reads process.env) cannot see, so a library caller passing // env: { GEMINI_API_KEYS: '...' } actually authenticates instead of hitting // 'No Gemini API keys configured'. Absent that, the pool handles selection. const keyOverride = resolveKeyOverrideFromEnv(input.env); const poolOptions: { fetcher?: typeof fetch; keyOverride?: string; } = {}; if (input.fetcher) poolOptions.fetcher = input.fetcher; if (keyOverride) poolOptions.keyOverride = keyOverride; const response = await fetchGeminiWithPool( buildTtsUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: requestBody, }, poolOptions, ); if (!response.ok) { const body = await response.text().catch(() => ''); throw new VclawError( 'tts_failed', `Gemini TTS request failed with HTTP ${response.status}.`, { backendId: 'gemini-tts', status: response.status, body: body.slice(0, 500) }, ); } let parsed: unknown; try { parsed = await response.json(); } catch (err) { throw new VclawError('tts_failed', 'Gemini TTS returned a non-JSON response.', { backendId: 'gemini-tts', error: err instanceof Error ? err.message : String(err), }); } const b64 = extractAudioBase64(parsed); if (!b64) { throw new VclawError('tts_failed', 'Gemini TTS response contained no audio data.', { backendId: 'gemini-tts', }); } const pcm = Buffer.from(b64, 'base64'); if (pcm.length === 0) { throw new VclawError('tts_failed', 'Gemini TTS returned an empty audio payload.', { backendId: 'gemini-tts', }); } const wav = wrapPcmInWav(pcm); await mkdir(dirname(input.outputPath), { recursive: true }); await writeFile(input.outputPath, wav); return { path: input.outputPath, durationMs: pcmDurationMs(pcm.length), backendId: 'gemini-tts', }; }, };