/** * Gemini STT for motion-overlay. * * `transcribeAudio` turns an audio file into a `Transcript` (segment-level * start/end/text) via the Gemini API, reusing the `gemini-key-pool` round-robin * + retry plumbing. The transport is injectable (`deps.fetcher`) so tests never * touch the network; a `deps.transcriber` override replaces the whole live path. * * The `--transcript ` operator override is handled by `loadTranscript`, * which loads + validates a bring-your-own transcript JSON of the same shape. * * Side-effect: reads the audio file + calls the Gemini API. Kept thin — the * shaping/validation is pure and unit-testable. */ import { readFile } from 'node:fs/promises'; import { extname } from 'node:path'; import { fetchGeminiWithPool } from '../gemini-key-pool.js'; import type { Transcript, TranscriptSegment } from './types.js'; const DEFAULT_STT_ENDPOINT = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent'; const STT_PROMPT = `You transcribe a single short spoken-word audio clip into timed segments. Return ONLY valid JSON with this exact shape (no markdown fences): { "language": "", "segments": [ { "start": , "end": , "text": "" } ] } Rules: - One segment per spoken sentence or natural breath group. - start/end are absolute seconds from the clip's 0:00, increasing, non-overlapping. - text is the spoken words verbatim, no speaker labels, no timestamps inside the text. - language is the detected spoken language as a BCP-47 code (e.g. "en", "pt", "es"). - Do not invent content; transcribe only what is spoken.`; /** A pluggable transcriber. Tests inject this to avoid the live Gemini path. */ export type TranscriberFn = (audioPath: string, language: string) => Promise; export interface TranscribeDeps { /** Replace the whole live transcription path (highest precedence). */ transcriber?: TranscriberFn; /** Inject a fetch for the live Gemini path (used when no transcriber given). */ fetcher?: typeof fetch; /** Override the Gemini endpoint (falls back to env then the default). */ endpoint?: string; } /** Map an audio file extension to a Gemini inlineData mime type. */ function audioMimeType(audioPath: string): string { switch (extname(audioPath).toLowerCase()) { case '.mp3': return 'audio/mp3'; case '.aac': return 'audio/aac'; case '.m4a': return 'audio/mp4'; case '.flac': return 'audio/flac'; case '.ogg': return 'audio/ogg'; case '.wav': default: return 'audio/wav'; } } /** Pull the joined text parts out of a Gemini generateContent response. */ function parseGeminiText(payload: unknown): string { const candidates = (payload as { candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }> }).candidates; const text = candidates?.[0]?.content?.parts ?.map((part) => part.text ?? '') .join('\n') .trim(); if (!text) { throw new Error('Gemini STT response did not contain text output.'); } return text; } /** * Validate + normalize an unknown value into a `Transcript`. Throws on any shape * violation. Pure — shared by the live path and the `--transcript` loader so the * contract is enforced in exactly one place. */ export function validateTranscript(value: unknown, source = 'transcript'): Transcript { if (!value || typeof value !== 'object') { throw new Error(`${source}: expected an object with { language, segments }.`); } const obj = value as { language?: unknown; segments?: unknown }; if (typeof obj.language !== 'string' || obj.language.trim() === '') { throw new Error(`${source}: "language" must be a non-empty string.`); } if (!Array.isArray(obj.segments)) { throw new Error(`${source}: "segments" must be an array.`); } const segments: TranscriptSegment[] = obj.segments.map((raw, i) => { if (!raw || typeof raw !== 'object') { throw new Error(`${source}: segment ${i} is not an object.`); } const seg = raw as { start?: unknown; end?: unknown; text?: unknown }; if (typeof seg.start !== 'number' || !Number.isFinite(seg.start) || seg.start < 0) { throw new Error(`${source}: segment ${i} "start" must be a finite, non-negative number.`); } if (typeof seg.end !== 'number' || !Number.isFinite(seg.end) || seg.end < seg.start) { throw new Error(`${source}: segment ${i} "end" must be a finite number >= start.`); } if (typeof seg.text !== 'string') { throw new Error(`${source}: segment ${i} "text" must be a string.`); } return { start: seg.start, end: seg.end, text: seg.text }; }); return { language: obj.language.trim(), segments }; } /** * Load + validate a bring-your-own transcript JSON (`--transcript `). * Throws on a missing/unreadable file or a shape violation. */ export async function loadTranscript(path: string): Promise { let raw: string; try { raw = await readFile(path, 'utf-8'); } catch (err) { throw new Error(`--transcript file not readable: ${path} (${(err as Error).message})`); } let parsed: unknown; try { parsed = JSON.parse(raw); } catch (err) { throw new Error(`--transcript file is not valid JSON: ${path} (${(err as Error).message})`); } return validateTranscript(parsed, `--transcript ${path}`); } /** * Transcribe an audio file into a `Transcript`. * * Precedence: * 1. `deps.transcriber` — full override (tests inject this; no network). * 2. live Gemini STT via `fetchGeminiWithPool` (uses `deps.fetcher` if given). * * @param audioPath path to the extracted audio clip. * @param language requested language hint ("auto" lets Gemini detect); the * returned transcript's `language` reflects the model's detection. */ export async function transcribeAudio( audioPath: string, language: string, deps: TranscribeDeps = {}, ): Promise { if (deps.transcriber) { return deps.transcriber(audioPath, language); } const endpoint = deps.endpoint ?? process.env.VCLAW_GEMINI_API_ENDPOINT ?? DEFAULT_STT_ENDPOINT; const audioBytes = await readFile(audioPath); const audioData = audioBytes.toString('base64'); const mimeType = audioMimeType(audioPath); const languageHint = language && language.toLowerCase() !== 'auto' ? `\nThe spoken language is "${language}"; transcribe in that language.` : ''; const response = await fetchGeminiWithPool( (key) => `${endpoint}${endpoint.includes('?') ? '&' : '?'}key=${encodeURIComponent(key)}`, { method: 'POST', headers: { 'Content-Type': 'application/json', Connection: 'close' }, body: JSON.stringify({ contents: [ { parts: [ { inlineData: { mimeType, data: audioData } }, { text: `${STT_PROMPT}${languageHint}` }, ], }, ], generationConfig: { temperature: 0, maxOutputTokens: 2048, responseMimeType: 'application/json', }, }), }, { ...(deps.fetcher ? { fetcher: deps.fetcher } : {}), onRetry: (label, status) => { process.stderr.write(`[motion-overlay/stt] ${label} returned HTTP ${status}; rotating key\n`); }, }, ); if (!response.ok) { throw new Error(`Gemini STT request failed with HTTP ${response.status}`); } const payload = await response.json(); const text = parseGeminiText(payload); const cleaned = text .replace(/^```json\s*/i, '') .replace(/^```\s*/i, '') .replace(/\s*```$/i, '') .trim(); let parsed: unknown; try { parsed = JSON.parse(cleaned); } catch (err) { throw new Error(`Gemini STT returned unparseable JSON: ${(err as Error).message}`); } return validateTranscript(parsed, 'Gemini STT'); }