import { ELEVENLABS_AGENTS_PAGE_SCHEMA, ELEVENLABS_CONVERSATION_SCHEMA, ELEVENLABS_CONVERSATIONS_PAGE_SCHEMA, ELEVENLABS_DUBBING_CREATED_SCHEMA, ELEVENLABS_DUBBING_SCHEMA, ELEVENLABS_FORCED_ALIGNMENT_SCHEMA, ELEVENLABS_MODEL_SCHEMA, ELEVENLABS_OUTBOUND_CALL_SCHEMA, ELEVENLABS_OUTPUT_FORMAT_SCHEMA, ELEVENLABS_TIMESTAMPED_AUDIO_SCHEMA, ELEVENLABS_TRANSCRIPTION_RESULT_SCHEMA, ELEVENLABS_USER_SCHEMA, ELEVENLABS_VOICE_SCHEMA, ELEVENLABS_VOICES_PAGE_SCHEMA, } from "@automate.ax/integration-contracts/elevenlabs" import * as z from "zod" import { defineAction } from "../../automation/actions" import { createElevenLabsForm, ELEVENLABS_ACCOUNT, extensionForElevenLabsFormat, getElevenLabsActionApi, } from "./lib" const VOICE_SETTINGS_SCHEMA = z.object({ similarityBoost: z.number().min(0).max(1).optional(), speed: z.number().min(0.7).max(1.2).optional(), stability: z.number().min(0).max(1).optional(), style: z.number().min(0).max(1).optional(), useSpeakerBoost: z.boolean().optional(), }) const MUSIC_OUTPUT_FORMAT_SCHEMA = z.enum([ "auto", "alaw_8000", "mp3_22050_32", "mp3_24000_48", "mp3_44100_32", "mp3_44100_64", "mp3_44100_96", "mp3_44100_128", "mp3_44100_192", "mp3_48000_128", "mp3_48000_192", "mp3_48000_240", "mp3_48000_320", "opus_48000_32", "opus_48000_64", "opus_48000_96", "opus_48000_128", "opus_48000_192", "pcm_8000", "pcm_16000", "pcm_22050", "pcm_24000", "pcm_32000", "pcm_44100", "pcm_48000", "ulaw_8000", ]) const SPEECH_INPUT_SCHEMA = z.object({ applyTextNormalization: z.enum(["auto", "off", "on"]).prefault("auto"), enableLogging: z.boolean().prefault(true), languageCode: z.string().min(2).max(3).optional(), modelId: z.string().min(1).prefault("eleven_multilingual_v2"), nextRequestIds: z.string().min(1).array().max(3).optional(), nextText: z.string().optional(), outputFormat: ELEVENLABS_OUTPUT_FORMAT_SCHEMA.prefault("mp3_44100_128"), previousRequestIds: z.string().min(1).array().max(3).optional(), previousText: z.string().optional(), seed: z.number().int().min(0).max(4_294_967_295).optional(), text: z.string().min(1), voiceId: z.string().min(1), voiceSettings: VOICE_SETTINGS_SCHEMA.optional(), }) /** Returns the ElevenLabs user and subscription represented by the key. */ export const getElevenLabsUser = defineAction("Get ElevenLabs user") .describe("Returns the connected ElevenLabs user and current plan usage.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(z.object({})) .output(ELEVENLABS_USER_SCHEMA) .retry({ replaySafety: "safe" }) .handler(({ account }) => getElevenLabsActionApi(account.secret).request("user", { responseSchema: ELEVENLABS_USER_SCHEMA, }), ) /** Lists models available to the connected ElevenLabs account. */ export const listElevenLabsModels = defineAction("List ElevenLabs models") .describe("Lists available models and their speech capabilities.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(z.object({})) .output(z.object({ models: ELEVENLABS_MODEL_SCHEMA.array() })) .retry({ replaySafety: "safe" }) .handler(async ({ account }) => ({ models: await getElevenLabsActionApi(account.secret).request("models", { responseSchema: ELEVENLABS_MODEL_SCHEMA.array(), }), })) /** Lists ElevenLabs voices with cursor pagination and provider filters. */ export const listElevenLabsVoices = defineAction("List ElevenLabs voices") .describe("Lists voices available to the connected account.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ category: z .enum(["cloned", "generated", "premade", "professional"]) .optional(), includeTotalCount: z.boolean().prefault(false), languages: z.string().min(2).max(3).array().max(100).optional(), nextPageToken: z.string().min(1).optional(), pageSize: z.number().int().min(1).max(100).prefault(20), search: z.string().min(1).optional(), sort: z.enum(["created_at_unix", "name"]).optional(), sortDirection: z.enum(["asc", "desc"]).optional(), voiceIds: z.string().min(1).array().max(100).optional(), voiceType: z .enum([ "community", "default", "non-community", "non-default", "personal", "saved", "workspace", ]) .optional(), }), ) .output(ELEVENLABS_VOICES_PAGE_SCHEMA) .retry({ replaySafety: "safe" }) .handler(({ account, input }) => { const { languages, ...query } = input return getElevenLabsActionApi(account.secret).request("v2/voices", { query: { ...query, language: languages }, responseSchema: ELEVENLABS_VOICES_PAGE_SCHEMA, }) }) /** Gets one ElevenLabs voice. */ export const getElevenLabsVoice = defineAction("Get ElevenLabs voice") .describe("Returns metadata for one available ElevenLabs voice.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(z.object({ voiceId: z.string().min(1) })) .output(ELEVENLABS_VOICE_SCHEMA) .retry({ replaySafety: "safe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).request( `voices/${encodeURIComponent(input.voiceId)}`, { responseSchema: ELEVENLABS_VOICE_SCHEMA }, ), ) /** Converts text to an audio file with an ElevenLabs voice. */ export const createElevenLabsSpeech = defineAction("Create ElevenLabs speech") .describe("Converts text to speech and returns the generated audio file.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(SPEECH_INPUT_SCHEMA) .output(z.instanceof(File)) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => { const { enableLogging, outputFormat, voiceId, ...body } = input return getElevenLabsActionApi(account.secret).requestFile( `text-to-speech/${encodeURIComponent(voiceId)}`, { body, filename: `elevenlabs-speech.${extensionForElevenLabsFormat(outputFormat)}`, method: "POST", query: { enableLogging, outputFormat }, }, ) }) /** Converts text to speech with character-level timing. */ export const createElevenLabsSpeechWithTimestamps = defineAction( "Create ElevenLabs speech with timestamps", ) .describe("Returns generated audio and character-level timing data.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(SPEECH_INPUT_SCHEMA) .output(ELEVENLABS_TIMESTAMPED_AUDIO_SCHEMA) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => { const { enableLogging, outputFormat, voiceId, ...body } = input return getElevenLabsActionApi(account.secret).request( `text-to-speech/${encodeURIComponent(voiceId)}/with-timestamps`, { body, method: "POST", query: { enableLogging, outputFormat }, responseSchema: ELEVENLABS_TIMESTAMPED_AUDIO_SCHEMA, }, ) }) /** Converts a multi-speaker script into one audio file. */ export const createElevenLabsDialogue = defineAction( "Create ElevenLabs dialogue", ) .describe( "Converts up to ten distinct voices and 2,000 characters into one dialogue file.", ) .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ applyTextNormalization: z.enum(["auto", "off", "on"]).prefault("auto"), inputs: z .object({ text: z.string().min(1), voiceId: z.string().min(1) }) .array() .min(1) .refine( (items) => new Set(items.map(({ voiceId }) => voiceId)).size <= 10, "ElevenLabs dialogue supports at most 10 distinct voices.", ) .refine( (items) => items.reduce((total, { text }) => total + text.length, 0) <= 2_000, "ElevenLabs dialogue supports at most 2,000 characters per request.", ), languageCode: z.string().min(2).max(3).optional(), modelId: z.string().min(1).prefault("eleven_v3"), outputFormat: ELEVENLABS_OUTPUT_FORMAT_SCHEMA.prefault("mp3_44100_128"), seed: z.number().int().min(0).max(4_294_967_295).optional(), }), ) .output(z.instanceof(File)) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => { const { outputFormat, ...body } = input return getElevenLabsActionApi(account.secret).requestFile( "text-to-dialogue", { body, filename: `elevenlabs-dialogue.${extensionForElevenLabsFormat(outputFormat)}`, method: "POST", query: { outputFormat }, }, ) }) /** Changes the voice in an audio file while preserving delivery. */ export const convertElevenLabsSpeech = defineAction("Convert ElevenLabs speech") .describe( "Changes the speaker in an audio file while preserving its content and delivery.", ) .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ audio: z.instanceof(File), fileFormat: z.enum(["other", "pcm_s16le_16"]).prefault("other"), modelId: z.string().min(1).prefault("eleven_english_sts_v2"), outputFormat: ELEVENLABS_OUTPUT_FORMAT_SCHEMA.prefault("mp3_44100_128"), removeBackgroundNoise: z.boolean().prefault(false), seed: z.number().int().min(0).max(4_294_967_295).optional(), voiceId: z.string().min(1), voiceSettings: VOICE_SETTINGS_SCHEMA.optional(), }), ) .output(z.instanceof(File)) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => { const { outputFormat, voiceId, ...formInput } = input return getElevenLabsActionApi(account.secret).requestFormFile( `speech-to-speech/${encodeURIComponent(voiceId)}`, createElevenLabsForm(formInput), { filename: `elevenlabs-converted-speech.${extensionForElevenLabsFormat(outputFormat)}`, query: { outputFormat }, }, ) }) /** Transcribes an audio or video file or a supported source URL. */ export const transcribeWithElevenLabs = defineAction( "Transcribe with ElevenLabs", ) .describe( "Transcribes an uploaded audio or video file or a supported source URL.", ) .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z .object({ detectSpeakerRoles: z.boolean().prefault(false), diarize: z.boolean().prefault(false), file: z.instanceof(File).optional(), fileFormat: z.enum(["other", "pcm_s16le_16"]).prefault("other"), keyterms: z.string().min(1).max(49).array().max(1_000).optional(), languageCode: z.string().min(2).max(3).optional(), modelId: z.string().min(1).prefault("scribe_v2"), multichannelOutputStyle: z .enum(["combined", "separate"]) .prefault("separate"), noVerbatim: z.boolean().prefault(false), numSpeakers: z.number().int().min(1).max(32).optional(), seed: z.number().int().min(0).max(2_147_483_647).optional(), sourceUrl: z.url().optional(), tagAudioEvents: z.boolean().prefault(true), temperature: z.number().min(0).max(2).optional(), timestampsGranularity: z .enum(["character", "none", "word"]) .prefault("word"), useMultiChannel: z.boolean().prefault(false), }) .refine(({ file, sourceUrl }) => Boolean(file) !== Boolean(sourceUrl), { message: "Provide exactly one of file or sourceUrl.", }), ) .output(ELEVENLABS_TRANSCRIPTION_RESULT_SCHEMA) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).requestForm( "speech-to-text", createElevenLabsForm(input), { responseSchema: ELEVENLABS_TRANSCRIPTION_RESULT_SCHEMA }, ), ) /** Generates a sound effect from a text description. */ export const createElevenLabsSoundEffect = defineAction( "Create ElevenLabs sound effect", ) .describe("Generates a sound effect from a text description.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ durationSeconds: z.number().min(0.5).max(30).optional(), loop: z.boolean().prefault(false), modelId: z.string().min(1).prefault("eleven_text_to_sound_v2"), outputFormat: ELEVENLABS_OUTPUT_FORMAT_SCHEMA.prefault("mp3_44100_128"), promptInfluence: z.number().min(0).max(1).prefault(0.3), text: z.string().min(1), }), ) .output(z.instanceof(File)) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => { const { outputFormat, ...body } = input return getElevenLabsActionApi(account.secret).requestFile( "sound-generation", { body, filename: `elevenlabs-sound-effect.${extensionForElevenLabsFormat(outputFormat)}`, method: "POST", query: { outputFormat }, }, ) }) /** Removes background sound from an audio or video file. */ export const isolateElevenLabsAudio = defineAction("Isolate ElevenLabs audio") .describe("Removes background noise, music, and ambience from speech.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ audio: z.instanceof(File), fileFormat: z.enum(["other", "pcm_s16le_16"]).prefault("other"), }), ) .output(z.instanceof(File)) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).requestFormFile( "audio-isolation", createElevenLabsForm(input), { filename: "elevenlabs-isolated-audio.mp3" }, ), ) /** Aligns a known transcript to an audio file. */ export const alignElevenLabsSpeech = defineAction("Align ElevenLabs speech") .describe( "Returns word and character timing for a known transcript and audio file.", ) .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(z.object({ file: z.instanceof(File), text: z.string().min(1) })) .output(ELEVENLABS_FORCED_ALIGNMENT_SCHEMA) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).requestForm( "forced-alignment", createElevenLabsForm(input), { responseSchema: ELEVENLABS_FORCED_ALIGNMENT_SCHEMA }, ), ) /** Composes a music track from a prompt. */ export const composeElevenLabsMusic = defineAction("Compose ElevenLabs music") .describe("Composes a complete music track from a text prompt.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ forceInstrumental: z.boolean().prefault(false), modelId: z .enum(["music_v1", "music_v2", "music_v2_5"]) .prefault("music_v1"), musicLengthMs: z.number().int().min(3_000).max(600_000).optional(), outputFormat: MUSIC_OUTPUT_FORMAT_SCHEMA.prefault("auto"), prompt: z.string().min(1).max(4_100), signWithC2pa: z.boolean().prefault(false), }), ) .output(z.instanceof(File)) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => { const { outputFormat, ...body } = input return getElevenLabsActionApi(account.secret).requestFile("music", { body, filename: `elevenlabs-music.${ outputFormat === "auto" ? "mp3" : extensionForElevenLabsFormat(outputFormat) }`, method: "POST", query: { outputFormat }, }) }) /** Creates an automatic dub from an audio/video file or source URL. */ export const createElevenLabsDubbing = defineAction("Create ElevenLabs dubbing") .describe("Starts an automatic dub for an uploaded file or source URL.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z .object({ disableVoiceCloning: z.boolean().prefault(false), dropBackgroundAudio: z.boolean().prefault(false), file: z.instanceof(File).optional(), highestResolution: z.boolean().prefault(false), name: z.string().min(1).optional(), numSpeakers: z.number().int().min(0).optional(), sourceLanguage: z .union([z.literal("auto"), z.string().min(2).max(3)]) .prefault("auto"), sourceUrl: z.url().optional(), targetAccent: z.string().min(1).optional(), targetLanguage: z.string().min(2).max(3), watermark: z.boolean().prefault(false), }) .refine(({ file, sourceUrl }) => Boolean(file) !== Boolean(sourceUrl), { message: "Provide exactly one of file or sourceUrl.", }), ) .output(ELEVENLABS_DUBBING_CREATED_SCHEMA) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => { const { sourceLanguage, targetLanguage, ...rest } = input return getElevenLabsActionApi(account.secret).requestForm( "dubbing", createElevenLabsForm({ ...rest, sourceLang: sourceLanguage, targetLang: targetLanguage, }), { responseSchema: ELEVENLABS_DUBBING_CREATED_SCHEMA }, ) }) /** Gets the processing state of an ElevenLabs dub. */ export const getElevenLabsDubbing = defineAction("Get ElevenLabs dubbing") .describe("Returns the current state and target languages for one dub.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(z.object({ dubbingId: z.string().min(1) })) .output(ELEVENLABS_DUBBING_SCHEMA) .retry({ replaySafety: "safe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).request( `dubbing/${encodeURIComponent(input.dubbingId)}`, { responseSchema: ELEVENLABS_DUBBING_SCHEMA }, ), ) /** Downloads one completed ElevenLabs dub. */ export const downloadElevenLabsDubbing = defineAction( "Download ElevenLabs dubbing", ) .describe("Downloads a completed dub as MP3 or MP4.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ dubbingId: z.string().min(1), languageCode: z.string().min(2).max(3), }), ) .output(z.instanceof(File)) .retry({ replaySafety: "safe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).requestFile( `dubbing/${encodeURIComponent(input.dubbingId)}/audio/${encodeURIComponent(input.languageCode)}`, { filename: `elevenlabs-dubbing-${input.languageCode}.mp3` }, ), ) /** Deletes one ElevenLabs dub. */ export const deleteElevenLabsDubbing = defineAction("Delete ElevenLabs dubbing") .describe("Permanently deletes one ElevenLabs dubbing project.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(z.object({ dubbingId: z.string().min(1) })) .output(z.object({ dubbingId: z.string().min(1), deleted: z.boolean() })) .retry({ replaySafety: "unsafe" }) .handler(async ({ account, input }) => ({ deleted: ( await getElevenLabsActionApi(account.secret).request( `dubbing/${encodeURIComponent(input.dubbingId)}`, { method: "DELETE", responseSchema: z.object({ status: z.string() }), }, ) ).status === "ok", dubbingId: input.dubbingId, })) /** Lists ElevenLabs agents available to the connected account. */ export const listElevenLabsAgents = defineAction("List ElevenLabs agents") .describe("Lists available ElevenLabs agents with cursor pagination.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ cursor: z.string().min(1).optional(), pageSize: z.number().int().min(1).max(100).prefault(30), search: z.string().min(1).optional(), }), ) .output(ELEVENLABS_AGENTS_PAGE_SCHEMA) .retry({ replaySafety: "safe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).request("convai/agents", { query: input, responseSchema: ELEVENLABS_AGENTS_PAGE_SCHEMA, }), ) /** Lists conversations handled by ElevenLabs agents. */ export const listElevenLabsConversations = defineAction( "List ElevenLabs conversations", ) .describe("Lists agent conversations with practical status and time filters.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ agentId: z.string().min(1).optional(), callStartAfterUnix: z.number().int().nonnegative().optional(), callStartBeforeUnix: z.number().int().nonnegative().optional(), callSuccessful: z .enum(["error", "failure", "success", "unknown"]) .optional(), cursor: z.string().min(1).optional(), pageSize: z.number().int().min(1).max(100).prefault(30), sortDirection: z.enum(["asc", "desc"]).prefault("desc"), userId: z.string().min(1).optional(), }), ) .output(ELEVENLABS_CONVERSATIONS_PAGE_SCHEMA) .retry({ replaySafety: "safe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).request("convai/conversations", { query: input, responseSchema: ELEVENLABS_CONVERSATIONS_PAGE_SCHEMA, }), ) /** Gets a complete ElevenLabs agent conversation. */ export const getElevenLabsConversation = defineAction( "Get ElevenLabs conversation", ) .describe("Returns one agent conversation and its transcript.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(z.object({ conversationId: z.string().min(1) })) .output(ELEVENLABS_CONVERSATION_SCHEMA) .retry({ replaySafety: "safe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).request( `convai/conversations/${encodeURIComponent(input.conversationId)}`, { responseSchema: ELEVENLABS_CONVERSATION_SCHEMA }, ), ) /** Downloads the audio for an ElevenLabs agent conversation. */ export const downloadElevenLabsConversationAudio = defineAction( "Download ElevenLabs conversation audio", ) .describe("Downloads the complete audio for one agent conversation.") .account("elevenlabs", ELEVENLABS_ACCOUNT) .input(z.object({ conversationId: z.string().min(1) })) .output(z.instanceof(File)) .retry({ replaySafety: "safe" }) .handler(({ account, input }) => getElevenLabsActionApi(account.secret).requestFile( `convai/conversations/${encodeURIComponent(input.conversationId)}/audio`, { filename: `elevenlabs-conversation-${input.conversationId}.mp3` }, ), ) /** Places an outbound call through an imported ElevenLabs Twilio number. */ export const createElevenLabsOutboundCall = defineAction( "Create ElevenLabs outbound call", ) .describe( "Places an outbound call through an ElevenLabs agent and imported Twilio number.", ) .account("elevenlabs", ELEVENLABS_ACCOUNT) .input( z.object({ agentId: z.string().min(1), agentPhoneNumberId: z.string().min(1), callRecordingEnabled: z.boolean().optional(), dynamicVariables: z .record(z.string(), z.union([z.boolean(), z.number(), z.string()])) .optional(), toNumber: z .string() .regex(/^\+[1-9]\d{1,14}$/, "Use an E.164 phone number."), }), ) .output(ELEVENLABS_OUTBOUND_CALL_SCHEMA) .retry({ replaySafety: "unsafe" }) .handler(({ account, input }) => { const { dynamicVariables, ...body } = input return getElevenLabsActionApi(account.secret).request( "convai/twilio/outbound-call", { body: { ...body, ...(dynamicVariables && { conversationInitiationClientData: { dynamicVariables }, }), }, method: "POST", responseSchema: ELEVENLABS_OUTBOUND_CALL_SCHEMA, }, ) })