/** * Music generation via Google Lyria 3 on the **Gemini Developer API** (NOT * Vertex). This is an API-KEY product, so it resolves a key from the existing * Gemini key pool (GEMINI_API_KEYS / GOOGLE_API_KEYS / GOOGLE_API_KEY) via * `fetchGeminiWithPool` — no Vertex project, no billing, just a Gemini key. * * VERIFIED LIVE (2026-06-01) against the Gemini Developer API: * * POST https://generativelanguage.googleapis.com/v1beta/models/{MODEL}:generateContent?key={KEY} * Content-Type: application/json * Body: { contents: [{ parts: [ { text: "" } , ] }] } * * Optional image-to-music conditioning attaches a second part in the SAME * parts array: * { inlineData: { mimeType: "image/png", data: "" } } * * Response: candidates[0].content.parts[] — find the part whose * inlineData.mimeType starts with "audio/" (observed "audio/mpeg"); its * inlineData.data is base64 MP3. (There is also a { text: "" } * part which we ignore.) * * MODEL (via LYRIA3_MODEL env, default lyria-3-clip-preview): * - lyria-3-clip-preview Lyria 3, 30s clip, text + image (default) * - lyria-3-pro-preview Lyria 3 Pro, full song up to ~184s, text + image * * Output is MP3 (audio/mpeg) — the file is always written with a `.mp3` * extension regardless of the requested outputPath suffix. durationMs is a * deterministic estimate from input.durationSec (or a model-default: 180s for * a 'pro' model, 30s otherwise) — true MP3 duration parsing is out of scope. * * Availability: a Gemini key is resolvable (pool size > 0 or one of the env * vars set); gated via the descriptor's requiresGeminiKey flag in the registry. */ import { mkdir, writeFile, readFile } from 'node:fs/promises'; import { dirname, extname } from 'node:path'; import { fetchGeminiWithPool } from '../gemini-key-pool.js'; import { VclawError } from '../errors.js'; import type { MusicBackend, MusicGenInput, MusicGenResult } from './types.js'; const DEFAULT_LYRIA3_MODEL = 'lyria-3-clip-preview'; /** Default clip lengths (seconds) when the caller omits durationSec. */ const PRO_DEFAULT_SEC = 180; const CLIP_DEFAULT_SEC = 30; /** * Gemini API key sources, in the same precedence as the global key pool * (gemini-key-pool.ts). Used to resolve an explicit key from a caller-supplied * `input.env` so the pool (which only reads process.env) can be bypassed. */ const GEMINI_KEY_VARS = ['GEMINI_API_KEYS', 'GOOGLE_API_KEYS', 'GOOGLE_API_KEY'] as const; /** * Resolve the first key from a caller-supplied env that process.env does not * already expose. Returns undefined when no env was passed, or when the passed * env adds no key beyond process.env (in which case the global pool selects). * Mirrors native-gemini-tts.ts's resolver. */ function resolveKeyOverrideFromEnv(env: NodeJS.ProcessEnv | undefined): string | undefined { if (!env || env === process.env) return undefined; for (const varName of GEMINI_KEY_VARS) { const raw = env[varName]; if (typeof raw !== 'string' || raw.trim() === '') continue; if (process.env[varName] === raw) continue; const first = raw.split(/[,;\n\s]+/).map((t) => t.trim()).find((t) => t.length > 0); if (first) return first; } return undefined; } /** Force a `.mp3` extension on the output path (Lyria 3 returns MP3). */ function forceMp3Path(p: string): string { const ext = extname(p); if (ext.toLowerCase() === '.mp3') return p; return ext ? `${p.slice(0, p.length - ext.length)}.mp3` : `${p}.mp3`; } /** Map a file extension to an image MIME type for the inlineData part. */ function imageMimeForPath(p: string): string { switch (extname(p).toLowerCase()) { case '.jpg': case '.jpeg': return 'image/jpeg'; case '.webp': return 'image/webp'; case '.gif': return 'image/gif'; case '.png': default: return 'image/png'; } } /** * Defensive extraction of the first base64 inlineData payload whose mimeType * starts with "audio/". The Gemini response also carries a { text } part (the * instrumental description) which must be ignored. */ function extractAudioBase64(parsed: unknown): string | undefined { if (typeof parsed !== 'object' || parsed === null) return undefined; const candidates = (parsed as { candidates?: unknown }).candidates; if (!Array.isArray(candidates) || candidates.length === 0) return undefined; const parts = (candidates[0] as { content?: { parts?: unknown } })?.content?.parts; if (!Array.isArray(parts)) return undefined; for (const part of parts) { const inlineData = (part as { inlineData?: { mimeType?: unknown; data?: unknown } })?.inlineData; const mime = inlineData?.mimeType; const data = inlineData?.data; if ( typeof mime === 'string' && mime.startsWith('audio/') && typeof data === 'string' && data.length > 0 ) { return data; } } return undefined; } export const lyria3Backend: MusicBackend = { id: 'lyria3', kind: 'music', displayName: 'Lyria 3 (Gemini API)', // API-key product, not Vertex. Availability is resolved from the Gemini key // pool (see isMusicBackendAvailable), so requiredEnv stays empty and the // registry gates on requiresGeminiKey instead of this backend's id. requiredEnv: [], requiresVertex: false, requiresGeminiKey: true, summary: 'Music generation via Google Lyria 3 on the Gemini Developer API (key-based, NOT Vertex). Default lyria-3-clip-preview (30s); LYRIA3_MODEL=lyria-3-pro-preview for a full song (~184s). Optional image-to-music conditioning via imagePath. Returns MP3. Requires a Gemini API key (GEMINI_API_KEYS / GOOGLE_API_KEYS / GOOGLE_API_KEY).', async generate(input: MusicGenInput): Promise { const env = input.env ?? process.env; const model = env['LYRIA3_MODEL'] ?? DEFAULT_LYRIA3_MODEL; const isPro = model.includes('pro'); const durationSec = input.durationSec ?? (isPro ? PRO_DEFAULT_SEC : CLIP_DEFAULT_SEC); const durationMs = durationSec * 1000; const outputPath = forceMp3Path(input.outputPath); if (!input.prompt.trim()) { throw new VclawError('music_gen_failed', 'Cannot generate music with an empty prompt.', { backend: 'lyria3', }); } // Dry-run: estimate duration from the model/request; do NOT touch network. // Write a placeholder MP3 (empty) so downstream existsSync() stays consistent. if (input.dryRun) { await mkdir(dirname(outputPath), { recursive: true }); await writeFile(outputPath, Buffer.alloc(0)); return { path: outputPath, durationMs, backendId: 'lyria3' }; } // Build the contents body: text prompt + optional image-to-music part. const parts: Array< { text: string } | { inlineData: { mimeType: string; data: string } } > = [{ text: input.prompt }]; if (input.imagePath) { let imgBytes: Buffer; try { imgBytes = await readFile(input.imagePath); } catch (err) { throw new VclawError( 'music_gen_failed', `Could not read image for image-to-music conditioning: ${input.imagePath}`, { backend: 'lyria3', error: err instanceof Error ? err.message : String(err) }, ); } parts.push({ inlineData: { mimeType: imageMimeForPath(input.imagePath), data: imgBytes.toString('base64'), }, }); } const requestBody = JSON.stringify({ contents: [{ parts }] }); // Resolve a key from input.env when it carries one the global pool (which // only reads process.env) cannot see; otherwise the pool selects. const keyOverride = resolveKeyOverrideFromEnv(input.env); const poolOptions: { fetcher?: typeof fetch; keyOverride?: string } = {}; if (input.fetcher) poolOptions.fetcher = input.fetcher; if (keyOverride) poolOptions.keyOverride = keyOverride; const response = await fetchGeminiWithPool( (key) => `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${encodeURIComponent(key)}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: requestBody, }, poolOptions, ); if (!response.ok) { const body = await response.text().catch(() => ''); throw new VclawError( 'music_gen_failed', `Lyria 3 (Gemini API) request failed with HTTP ${response.status}.`, { backend: 'lyria3', status: response.status, body: body.slice(0, 500) }, ); } let parsed: unknown; try { parsed = await response.json(); } catch (err) { throw new VclawError('music_gen_failed', 'Lyria 3 returned a non-JSON response.', { backend: 'lyria3', error: err instanceof Error ? err.message : String(err), }); } const b64 = extractAudioBase64(parsed); if (!b64) { throw new VclawError( 'music_gen_failed', 'Lyria 3 response contained no audio/* inlineData part.', { backend: 'lyria3' }, ); } const bytes = Buffer.from(b64, 'base64'); if (bytes.length === 0) { throw new VclawError('music_gen_failed', 'Lyria 3 returned an empty audio payload.', { backend: 'lyria3', }); } await mkdir(dirname(outputPath), { recursive: true }); await writeFile(outputPath, bytes); // durationMs is an estimate from the request/model (MP3 duration parsing is // out of scope — Lyria 3 returns audio/mpeg without a trivial header read). return { path: outputPath, durationMs, backendId: 'lyria3' }; }, };