/** * Gemini Interactions API client (`POST /v1beta/interactions`) — the surface * that carries agentic video understanding (`processing: "agentic"` on a * video input: the model reads the video by reference and fetches transcript, * frame windows and audio on demand instead of ingesting the whole file). * * Contract live-verified 2026-09-16, see * `docs/audits/2026-09-16-gemini-agentic-contract.md`: * - `?key=` auth works; a YouTube `uri` needs no `mime_type`; inline `data` * is accepted under agentic. * - The response array is `steps` (kept tolerant of `outputs`); only * `type: "model_output"` items carry `content[].text`. Other step types carry * opaque `signature` blobs that must never be copied into an artifact. * - `status: "incomplete"` means the per-invocation output cap was hit — and * thinking tokens count against `max_output_tokens` — so it is surfaced as * its own error and never retried. * - `id` is null with `store: false`, so there is nothing to poll; v1 is * synchronous only. `generation_config.response_format` is rejected. */ import { fetchGeminiWithPool } from './gemini-key-pool.js'; import { resolveGeminiApiBase, withKey } from './gemini-api-base.js'; import { safeErrorBody } from './http-error-safety.js'; export const DEFAULT_AGENTIC_MODEL = 'gemini-3.8-flash'; /** Models Google lists as supporting agentic video processing (2026-09-16). */ export const AGENTIC_MODELS: readonly string[] = [ 'gemini-3.8-flash', 'gemini-3.7-flash', 'gemini-3.6-flash', 'gemini-3.5-flash-lite', ]; /** Allowlist check; `VCLAW_GEMINI_AGENTIC_MODELS=a,b` extends the built-in list. */ export function isAgenticModel(model: string, env: NodeJS.ProcessEnv = process.env): boolean { const extra = (env.VCLAW_GEMINI_AGENTIC_MODELS ?? '') .split(/[,\s]+/) .map((id) => id.trim()) .filter((id) => id !== ''); return AGENTIC_MODELS.includes(model) || extra.includes(model); } export function isYouTubeUrl(source: string): boolean { return /^https?:\/\/(www\.|m\.|music\.)?(youtube\.com\/(watch\?|shorts\/|live\/|embed\/|v\/)|youtu\.be\/)\S+/i.test(source.trim()); } export type InteractionProcessing = 'static' | 'agentic'; export interface InteractionVideoInput { uri?: string; /** Base64 bytes — the alternative to `uri` for small clips. */ data?: string; mimeType?: string; processing: InteractionProcessing; } export interface InteractionUsage { inputTokens?: number; outputTokens?: number; totalTokens?: number; thoughtTokens?: number; toolUseTokens?: number; cachedTokens?: number; byModality?: Array<{ modality: string; tokens: number }>; } export interface InteractionResult { id?: string; model: string; status: string; text: string; usage?: InteractionUsage; } type Step = { type?: string; content?: Array<{ type?: string; text?: string }> }; /** Joins the text of every `model_output` step; accepts `steps` or `outputs`. */ export function extractInteractionText(payload: unknown): string { const record = (payload ?? {}) as { steps?: Step[]; outputs?: Step[] }; const items = Array.isArray(record.steps) ? record.steps : Array.isArray(record.outputs) ? record.outputs : []; const text = items .filter((step) => step?.type === 'model_output' && Array.isArray(step.content)) .flatMap((step) => step.content ?? []) .filter((part) => (part.type ?? 'text') === 'text' && typeof part.text === 'string') .map((part) => part.text as string) .join('\n') .trim(); if (!text) { throw new Error( `Gemini interactions response carried no model_output text (top-level keys: ${Object.keys(record).sort().join(', ')}).`, ); } return text; } function numberOr(value: unknown): number | undefined { return typeof value === 'number' && Number.isFinite(value) ? value : undefined; } export function normalizeInteractionUsage(raw: unknown): InteractionUsage | undefined { if (!raw || typeof raw !== 'object') return undefined; const u = raw as Record; const byModality = Array.isArray(u.input_tokens_by_modality) ? (u.input_tokens_by_modality as Array<{ modality?: string; tokens?: number }>) .filter((row) => typeof row?.modality === 'string' && typeof row?.tokens === 'number') .map((row) => ({ modality: row.modality as string, tokens: row.tokens as number })) : undefined; const usage: InteractionUsage = { ...(numberOr(u.total_input_tokens) !== undefined ? { inputTokens: numberOr(u.total_input_tokens) } : {}), ...(numberOr(u.total_output_tokens) !== undefined ? { outputTokens: numberOr(u.total_output_tokens) } : {}), ...(numberOr(u.total_tokens) !== undefined ? { totalTokens: numberOr(u.total_tokens) } : {}), ...(numberOr(u.total_thought_tokens) !== undefined ? { thoughtTokens: numberOr(u.total_thought_tokens) } : {}), ...(numberOr(u.total_tool_use_tokens) !== undefined ? { toolUseTokens: numberOr(u.total_tool_use_tokens) } : {}), ...(numberOr(u.total_cached_tokens) !== undefined ? { cachedTokens: numberOr(u.total_cached_tokens) } : {}), ...(byModality && byModality.length > 0 ? { byModality } : {}), }; return Object.keys(usage).length > 0 ? usage : undefined; } function payloadError(payload: Record): string { const error = payload.error; if (typeof error === 'string') return error; if (error && typeof error === 'object') { const message = (error as { message?: unknown }).message; if (typeof message === 'string') return message; return safeErrorBody(JSON.stringify(error)); } return ''; } /** * The exact request body `createInteraction` sends. Extracted so a `--dry-run` * can print the real payload instead of a hand-copied shape that drifts. * PURE — no network, no key. */ export function buildInteractionRequestBody(input: { model: string; video: InteractionVideoInput; text: string; /** Passed through verbatim; the API wants snake_case keys. */ generationConfig?: Record; }): Record { const video: Record = { type: 'video', processing: input.video.processing }; if (input.video.uri) video.uri = input.video.uri; if (input.video.data) video.data = input.video.data; if (input.video.mimeType) video.mime_type = input.video.mimeType; return { model: input.model, input: [video, { type: 'text', text: input.text }], ...(input.generationConfig ? { generation_config: input.generationConfig } : {}), store: false, }; } export async function createInteraction(input: { model: string; video: InteractionVideoInput; text: string; /** Passed through verbatim; the API wants snake_case keys. */ generationConfig?: Record; apiBase?: string; fetcher?: typeof fetch; keyOverride?: string; /** Stderr prefix for retry notices; defaults to `[analyze/gemini]`. */ logPrefix?: string; }): Promise { const apiBase = input.apiBase ?? resolveGeminiApiBase(); const prefix = input.logPrefix ?? '[analyze/gemini]'; const body = buildInteractionRequestBody({ model: input.model, video: input.video, text: input.text, ...(input.generationConfig ? { generationConfig: input.generationConfig } : {}), }); const response = await fetchGeminiWithPool( (key) => withKey(`${apiBase}/v1beta/interactions`, key), { method: 'POST', headers: { 'Content-Type': 'application/json', 'Connection': 'close' }, body: JSON.stringify(body), }, { fetcher: input.fetcher, keyOverride: input.keyOverride, // A 5xx retry after the server already processed the video re-spends; // allow exactly one retry. maxAttempts: 2, onRetry: (label, status) => { process.stderr.write(`${prefix} ${label} returned HTTP ${status} on /interactions; retrying once\n`); }, }, ); if (!response.ok) { const text = await response.text().catch(() => ''); throw new Error(`Gemini interactions request failed with HTTP ${response.status}: ${safeErrorBody(text)}`); } let payload: Record; try { payload = (await response.json()) as Record; } catch { throw new Error('Gemini interactions request returned a 2xx response with an unparseable JSON body.'); } const status = typeof payload.status === 'string' ? payload.status : 'unknown'; const id = typeof payload.id === 'string' ? payload.id : undefined; if (status === 'incomplete') { throw new Error( 'Gemini interaction stopped with status "incomplete": the output cap was hit (thinking tokens count against max_output_tokens per model invocation). Raise max_output_tokens.', ); } if (status !== 'completed') { const detail = payloadError(payload); throw new Error(`Gemini interaction ended with status "${status}"${detail ? `: ${detail}` : ''}.`); } return { ...(id ? { id } : {}), model: typeof payload.model === 'string' ? payload.model : input.model, status, text: extractInteractionText(payload), ...(normalizeInteractionUsage(payload.usage) ? { usage: normalizeInteractionUsage(payload.usage) } : {}), }; }