/** * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import { ActionMetadata, GenerateResponseData, Operation, modelActionMetadata, z, } from 'genkit'; import { BackgroundModelAction, modelRef, type GenerateRequest, type ModelInfo, type ModelReference, } from 'genkit/model'; import { backgroundModel as pluginBackgroundModel } from 'genkit/plugin'; import { isKnownKey } from '../common/utils.js'; import { veoCheckOperation, veoPredict } from './client.js'; import { ClientOptions, GoogleAIPluginOptions, Model, VeoOperation, VeoParameters, VeoPredictRequest, } from './types.js'; import { applyContextOverrides, calculateApiKey, checkModelName, extractText, extractVeoImage, extractVeoVideo, extractVersion, modelName, } from './utils.js'; /** * See https://ai.google.dev/gemini-api/docs/video */ export const VeoConfigSchema = z .object({ negativePrompt: z.string().optional(), aspectRatio: z .enum(['9:16', '16:9']) .describe('Desired aspect ratio of the output video.') .optional(), personGeneration: z .enum(['dont_allow', 'allow_adult', 'allow_all']) .describe( 'Control if/how images of people will be generated by the model.' ) .optional(), durationSeconds: z .number() .step(1) .min(4) // Veo 3.1 supports 4s for some resolutions. .max(8) .describe('Length of each output video in seconds.') .optional(), resolution: z .enum(['720p', '1080p', '4k']) .describe('Resolution of the output video.') .optional(), seed: z .number() .int() .describe('Random seed for the video generation.') .optional(), enhancePrompt: z .boolean() .describe('Enable or disable the prompt rewriter. Enabled by default.') .optional(), }) .passthrough(); export type VeoConfigSchemaType = typeof VeoConfigSchema; export type VeoConfig = z.infer; // This contains all the Veo config schema types type ConfigSchemaType = VeoConfigSchemaType; function commonRef( name: string, info?: ModelInfo, configSchema: ConfigSchemaType = VeoConfigSchema ): ModelReference { return modelRef({ name: `googleai/${name}`, configSchema, info: info ?? ({ supports: { media: true, multiturn: false, tools: false, systemRole: false, output: ['media'], longRunning: true, }, } as ModelInfo), // TODO(ifielker): Remove this cast if we fix longRunning }); } const GENERIC_MODEL = commonRef('veo'); const KNOWN_MODELS = {} as const; export type KnownModels = keyof typeof KNOWN_MODELS; // For autocomplete // Models we no longer suggest, but that still work: they still resolve and // are still registered as actions, but aren't in the KnownModels type, so // they don't show up in autocomplete. Set `stage` on each one (keeping the // default `supports`): 'deprecated' for models that are deprecated or // < 1 month from shutdown. const HIDDEN_MODELS = { // Shut down on Oct 22 2026. 'veo-3.1-lite-generate-preview': commonRef('veo-3.1-lite-generate-preview', { ...GENERIC_MODEL.info, stage: 'deprecated', }), 'veo-3.1-generate-preview': commonRef('veo-3.1-generate-preview', { ...GENERIC_MODEL.info, stage: 'deprecated', }), 'veo-3.1-fast-generate-preview': commonRef('veo-3.1-fast-generate-preview', { ...GENERIC_MODEL.info, stage: 'deprecated', }), } as const; const ALL_MODELS = { ...HIDDEN_MODELS, ...KNOWN_MODELS, }; export type VeoModelName = `veo-${string}`; export function isVeoModelName(value?: string): value is VeoModelName { return !!value?.startsWith('veo-'); } export function model( version: string, config: VeoConfig = {} ): ModelReference { const name = checkModelName(version); if (isKnownKey(name, ALL_MODELS)) { return ALL_MODELS[name].withConfig(config); } return modelRef({ name: `googleai/${name}`, config, configSchema: VeoConfigSchema, info: { ...GENERIC_MODEL.info }, }); } // Takes a full list of models, filters for current Veo models only // and returns a modelActionMetadata for each. export function listActions(models: Model[]): ActionMetadata[] { return ( models .filter( (m) => m.supportedGenerationMethods.includes('predictLongRunning') && isVeoModelName(modelName(m.name)) ) // Filter out deprecated .filter((m) => !m.description || !m.description.includes('deprecated')) .map((m) => { const ref = model(m.name); return modelActionMetadata({ name: ref.name, info: ref.info, configSchema: ref.configSchema, }); }) ); } export function listKnownModels(options?: GoogleAIPluginOptions) { return Object.keys(ALL_MODELS).map((name: string) => defineModel(name, options) ); } /** * Defines a new GoogleAI Veo model. */ export function defineModel( name: string, pluginOptions?: GoogleAIPluginOptions ): BackgroundModelAction { const ref = model(name); const clientOptions: ClientOptions = { apiVersion: pluginOptions?.apiVersion, baseUrl: pluginOptions?.baseUrl, customHeaders: pluginOptions?.customHeaders, experimental_debugTraces: pluginOptions?.experimental_debugTraces, }; return pluginBackgroundModel({ name: ref.name, ...ref.info, configSchema: ref.configSchema, async start(request, options) { // apiKey in config is deprecated in favor of options.context.secrets.apiKey const newClientOptions = applyContextOverrides( { ...clientOptions, apiKey: calculateApiKey( pluginOptions?.apiKey, request.config?.apiKey as string | undefined ), }, options?.context ); const apiKey = newClientOptions.apiKey; const veoPredictRequest: VeoPredictRequest = { instances: [ { prompt: extractText(request), image: extractVeoImage(request), video: extractVeoVideo(request), }, ], parameters: toVeoParameters(request), }; const response = await veoPredict( apiKey, extractVersion(ref), veoPredictRequest, newClientOptions ); return fromVeoOperation(response); }, async check(operation, options) { const checkOptions = applyContextOverrides( { ...clientOptions, apiKey: calculateApiKey(pluginOptions?.apiKey, undefined), }, options?.context ); const response = await veoCheckOperation( checkOptions.apiKey, operation.id, checkOptions ); return fromVeoOperation(response); }, }); } function toVeoParameters( request: GenerateRequest ): VeoParameters { const out = { ...request?.config, }; for (const k in out) { // undefined is handled by JSON.stringify // false is needed so we can set enhancePrompt to false if (out[k] === null) delete out[k]; } // This is not part of the request parameters sent to the endpoint // It's pulled out and used separately delete out.apiKey; // This was used to help us figure out which model. We no longer need // it here. delete out.version; return out; } function fromVeoOperation( apiOp: VeoOperation ): Operation { const res = { id: apiOp.name } as Operation; if (apiOp.done !== undefined) { res.done = apiOp.done; } if (apiOp.error) { res.error = { message: apiOp.error.message }; } if ( apiOp.response && apiOp.response.generateVideoResponse && apiOp.response.generateVideoResponse.generatedSamples ) { res.output = { finishReason: 'stop', raw: apiOp.response, message: { role: 'model', content: apiOp.response.generateVideoResponse.generatedSamples.map( (s) => { return { media: { url: s.video.uri, }, }; } ), }, }; } return res; } export const TEST_ONLY = { toVeoParameters, fromVeoOperation, GENERIC_MODEL, KNOWN_MODELS, HIDDEN_MODELS, ALL_MODELS, };