import { AliRtcCheckResult } from 'aliyun-rtc-sdk'; import { default as default_2 } from 'eventemitter3'; import { default as default_3 } from 'aliyun-rtc-sdk'; /** * 通话环境参数 * Ambient Config */ export declare class AICallAgentAmbientConfig { /** * 背景音音量,默认为100 */ volume: number; /** * 控制台注册背景音后的resourceid,空字符串表示关闭。 */ resourceId?: string; /** * 初始化(通过toJSON方法生成的字典) * Initialization (via a dictionary generated by the toJSON method) */ fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } export declare class AICallAgentAsrConfig { /** * asr语种Id,为空表示使用智能体配置值 * ASR language ID. If empty, the agent's configured value will be used. */ asrLanguageId: string | undefined; /** * 语音断句检测阈值,静音时长超过该阈值会被认为断句,参数范围 200ms~1200ms,默认值:-1:小于0表示跟随智能体默认配置(使用控制台配置值) * Voice sentence detection threshold. Silence exceeding this threshold will be considered as a sentence break. The range is 200ms to 1200ms. */ asrMaxSilence: number; /** * ASR热词列表 * Hot words for ASR */ asrHotWords: string[] | undefined; /** * 设置AIVad的灵敏度参数,通过该参数的设定可以抗人声干扰。取值范围[0-11],0表示关闭AIVad,服务端默认值是0,客户端默认开启AIVad且值为11 * Set the sensitivity parameter for AIVad. By setting this parameter, resistance to human voice interference can be achieved. The valid range is [0-10]. A value of 0 means AIVad is disabled. The server default value is 0, but the client enables AIVad by default with a value of 11. */ vadLevel: number; /** * 语音活动检测的最短持续时间阈值,用于控制打断的灵敏度。默认为0表示关闭此功能;有效范围:200到2000毫秒,常用[200,500] 对应1-4个字。,如果设置小于0则不会把该值发给服务端(服务端默认关闭此功能) * Minimum VAD Duration (ms): Controls how long voice activity must persist before being recognized. Default: 0 (disabled), Range: 200–2000 ms, Recommended: 200–500 ms. Negative values disable sending this parameter to the server. */ vadDuration: number; /** * 自接入ASR时,传入运行时参数,使用URL参数形式,例如:"mode=fast&sample=16000&format=wav" * When integrating with ASR manually, pass runtime parameters in URL query format. */ customParams?: string; fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } export declare class AICallAgentAutoSpeechConfig { llmPending?: AICallAgentAutoSpeechLlmPending; userIdle?: AICallAgentAutoSpeechUserIdle; fromJSON(data: JSONObject): void; toJSON(): JSONObject; } /** * 智能体在自动讲话场景下的讲话内容(包括:附和语、主动问询等) */ export declare class AICallAgentAutoSpeechContent { text: string; probability: number; fromJSON(data: JSONObject): void; toJSON(): JSONObject; } /** * 智能体在LLM 响应延迟时的播报自动讲话的配置 */ export declare class AICallAgentAutoSpeechLlmPending { waitTime: number; messages: AICallAgentAutoSpeechContent[]; fromJSON(data: JSONObject): void; toJSON(): JSONObject; } /** * 智能体在用户静默时问询的配置 */ export declare class AICallAgentAutoSpeechUserIdle { /** * 静默时长阈值,毫秒。超过此时长触发问询。范围 5000–600000ms,建议10000. */ waitTime: number; /** * 最大问询次数,范围 0–10,建议5。超过后不再触发,直接关闭通话。 */ maxRepeats: number; /** * 等待提示语集合,最大 10 条,每条长度 ≤ 100 字符,概率总和为1.0。 */ messages: AICallAgentAutoSpeechContent[]; fromJSON(data: JSONObject): void; toJSON(): JSONObject; } export declare class AICallAgentAvatarConfig { /** * 数字人模型Id,为空表示使用智能体配置值 * Digital human model ID. If empty, the agent's configured value will be used. */ agentAvatarId: string | undefined; fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } export declare class AICallAgentBackChanneling { /** * 是否启用附和语功能 * Whether to enable back channeling */ enable: boolean; /** * 附和触发的时机 */ triggerStage: string; /** * 触发概率。范围 0.0–1.0 */ probability: number; /** * 等待提示语集合,最大 10 条,每条长度 ≤ 100 字符,概率总和为1.0。 */ words: AICallAgentAutoSpeechContent[]; fromJSON(data: JSONObject): void; toJSON(): JSONObject; } /** * 通话智能体启动与运行配置 * Configuration For Agent Startup & Running */ export declare class AICallAgentConfig { /** * 智能体欢迎语,为空表示使用智能体配置值 * Agent greeting message. If empty, the agent's configured value will be used. */ agentGreeting: string | undefined; /** * 用户在通话启动前的指令,用于智能体在通话启动后立即响应,例如:“今天天气怎么样?” * User instruction before the call starts, used for the agent to respond immediately after the call starts, e.g., "How's the weather today?" */ wakeUpQuery: string | undefined; /** * 智能体闲时的最大等待时间(单位:秒),超时智能体自动下线,小于0则使用服务端默认值600s * Maximum idle time for the agent (in seconds). If the timeout is reached, the agent will automatically go offline. If less than 0, the server default value of 600 seconds will be used. */ agentMaxIdleTime: number; /** * 用户未入会,智能体超时关闭任务的时间,默认值为60s,小于0则不会发给服务端(使用服务端默认值60s) * Timeout for the agent to close the task if the user hasn't joined the meeting. If less than 0, the server default value of 60 seconds will be used. * */ userOnlineTimeout: number; /** * 用户退会后,智能体超时关闭任务的时间,默认值为5s,小于0则不会发给服务端(使用服务端默认值5s) * Timeout for the agent to close the task after the user leaves the meeting. If less than 0, the server default value of 5 seconds will be used. */ userOfflineTimeout: number; /** * 是否开启对讲机模式 * Whether to enable walkie-talkie mode */ enablePushToTalk: boolean; /** * 是否优雅下线 * 优雅下线:当智能体被停止的时候,播报完当前说的话再停止,最多播报 10 秒 * Whether to enable graceful shutdown. Graceful shutdown means that when the agent is stopped, it will finish saying the current sentence before stopping, with a maximum duration of 10 seconds. */ agentGracefulShutdown: boolean; /** * 智能体说话的音量,范围为 0~400,输出音量=工作流中的语音输出音量 * volume/100,默认值为100,小于0则不会发给服务端(使用服务端默认值100) * Agent's speaking volume. The range is 0 to 400. Output volume = voice output volume in the workflow * volume / 100. If less than 0, the server default value of 100 will be used. */ volume: number; /** * 工作流覆盖参数 * Workflow override parameters */ workflowOverrideParams: JSONObject | undefined; /** * 智能断句开关,开启智能断句后,用户说话的发生断句会智能合并成一句 * Intelligent sentence segmentation switch. When enabled, user speech breaks will be intelligently merged into one sentence. */ enableIntelligentSegment: boolean; /** * 接通后,播放欢迎语前的音效,支持url传入;播放完音效之后,欢迎语还是会播放。 */ preConnectAudioUrl?: string; /** * 语音识别配置 * ASR configuration */ asrConfig: AICallAgentAsrConfig; /** * 语音合成配置 * TTS configuration */ ttsConfig: AICallAgentTtsConfig; /** * 大语言模型配置 * LLM configuration */ llmConfig: AICallAgentLlmConfig; /** * 数字人配置 * Avatar configuration */ avatarConfig: AICallAgentAvatarConfig; /** * 打断配置 * Interruption configuration */ interruptConfig: AICallAgentInterruptConfig; /** * 声纹降噪配置 * Voiceprint configuration */ voiceprintConfig: AICallAgentVoiceprintConfig; /** * 轮次检测配置 * Turn detection configuration */ turnDetectionConfig: AICallAgentTurnDetectionConfig; /** * 非产品化定制化配置 * Experimental configuration */ experimentalConfig?: JSONObject; /** * 视觉理解配置 * Vision configuration */ vcrConfig: AICallAgentVcrConfig; /** * 通话环境参数 * Ambient Config */ ambientConfig: AICallAgentAmbientConfig; /** * 附和语功能配置模块。配置后,系统会在特定触发时机随机播报简短的附和语 */ backChannelingConfig?: AICallAgentBackChanneling[]; /** * 智能体自动讲话配置 * llmPending: 智能体在LLM 响应延迟时的播报自动讲话的配置 * userIdle: 智能体在用户静默时问询的配置 */ autoSpeechConfig: AICallAgentAutoSpeechConfig; constructor(); fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } export declare class AICallAgentError extends Error { code?: number; constructor(message: string, code?: number); } export declare type AICallAgentInfo = { agentType?: AICallAgentType; instanceId: string; channelId: string; /** * RTC UserId of Agent */ userId: string; rtcToken: string; reqId?: string; }; export declare type AICallAgentInfoAPIResp = { AIAgentUserId: string; ChannelId: string; InstanceId: string; RequestId: string; Token: string; }; export declare type AICallAgentInfoResp = { ai_agent_instance_id: string; ai_agent_user_id: string; rtc_auth_token: string; channel_id: string; request_id: string; }; export declare class AICallAgentInterruptConfig { /** * 是否开启智能打断 * Whether to enable intelligent interruption */ enableVoiceInterrupt: boolean; /** * 触发对话打断的特定词汇或短语,例如:“打断一下”、“我知道了” * Specific words or phrases that trigger conversation interruption, e.g., "Interrupt," "I know." */ interruptWords: string[] | undefined; /** * 智能体不开启智能打断且在讲话时,控制此时用户侧讲话的ASR文本处理策略,有以下取值: * “cache”:缓存ASR文本,当前轮次结束后,被缓存的asr文本会在下一个轮次一起被处理 * "discard":直接丢弃ASR文本 * 其他值(包括空值):走服务端默认配置 */ noInterruptMode: string | undefined; fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } export declare class AICallAgentLlmConfig { /** * llm/mllm 历史对话上下文最大保留轮次,默认值为-1,小于0表示跟随智能体默认配置(使用控制台配置值) * The maximum number of rounds for retaining historical conversation context in llm/mllm. */ llmHistoryLimit: number; /** * llm的系统提示词,例如:“你是一位友好且乐于助人的助手,专注于为用户提供准确的信息和建议。” * LLM system prompt, e.g., "You are a friendly and helpful assistant, focused on providing accurate information and advice to users." */ llmSystemPrompt: string | undefined; /** * 百炼应用中心参数 * Bailian App Center parameters */ bailianAppParams: JSONObject | undefined; /** * 是否发送完整的LLM结果。开启后,LLM生成结果之后,会通过onLLMReplyCompleted事件回调完整llm结果。 * Whether to enable sending the all LLM reply text. The 'onLLMReplyCompleted' event will be raised if enable. */ llmCompleteReply: boolean; /** * 额外的 OpenAI 协议 LLM 的查询参数,参数必须使用 key=value 格式,多个参数用 & 连接,所有值必须为字符串类型。 * Additional query parameters for the OpenAI protocol-based LLM. All parameters must follow the key=value format, multiple parameters should be concatenated using &, and all values must be strings. * Example: api-version=2024-02-01&api-key=sk-xxx */ openAIExtraQuery?: string; /** * 文本输出最小长度(字符数),小于该长度的文本会被缓存等待拼接,范围[0, 100],小于或等于0表示不限制,默认不限制。 */ outputMinLength: number; /** * 文本输出最大延迟时间(毫秒),超过该时间强制输出已缓存文本,。范围[1000,10000],小于或等于0表示不限制,默认不限制。 */ outputMaxDelay: number; /** * 大模型消息历史保存,是否和tts播放内容保持一致。默认false。开启后,保存的大模型消息和tts播放内容保持一致,允许有轻微误差。 */ historySyncWithTTS: boolean; fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } export declare class AICallAgentShareConfig { /** * 智能体分享ID */ shareId?: string; /** * 智能体工作量类型 */ agentType?: AICallAgentType; /** * 过期时间 */ expireTime?: Date; /** * 模板配置(Json字符串) */ templateConfig?: string; /** * 服务所在区域 */ region?: string; /** * 用户配置(Json字符串) */ userData?: string; constructor(shareId: string, agentType: AICallAgentType, expireTime: Date, templateConfig: string, region: string, userData?: string); static fromToken(token: string): AICallAgentShareConfig; } export declare enum AICallAgentState { /** * 聆听中 */ Listening = 1, /** * 思考中 */ Thinking = 2, /** * 讲话中 */ Speaking = 3 } export declare class AICallAgentTtsConfig { /** * 智能体讲话音色Id,为空表示使用智能体配置值 * Agent's voice tone ID. If empty, the agent's configured value will be used. */ agentVoiceId: string | undefined; /** * 发音规则数组,最多支持 20 条规则。若为 undefined 或空数组,则不使用任何规则。 * An array containing up to 20 custom pronunciation rules. If this value is nil or an empty array, no pronunciation rules will be used. * * 此 API 用于设置自定义的中文发音规则,适用于 TTS(Text-To-Speech)引擎在朗读文本时替换某些词的默认发音。 * 例如: * “重(chóng)叠” → 设置为读作“chǒng叠” * “行(xíng)动” → 设置为读作“háng动” * 调用示例: * ttsConfig.pronunciationRules = [ * { * "Word": "重叠", // 目标词 * "Pronunciation": "宠叠", // 替换发音 * "Type": "replacement" // 多音字规则 * }, * { * "Word": "行动", * "Pronunciation": "航动", * "Type": "replacement" * } * ] */ pronunciationRules?: JSONObject[]; /** * TTS播报语速,支持所有TTS类型,取值范围为[0.5, 2.0],默认值为-1,小于0表示跟随智能体默认配置(使用控制台配置值) * TTS speech rate, supported by all TTS types, value range is [0.5, 2.0]. */ speechRate: number; /** * TTS播报语种代码,当TTS类型为MiniMax有效 * TTS language id, valid when TTS type is MiniMax. * */ languageId?: string; /** * TTS播报情绪类型,当TTS类型为MiniMax有效 * TTS emotion type, valid when TTS type is MiniMax. * */ emotion?: string; fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } export declare class AICallAgentTurnDetectionConfig { /** * 断句特定词,例如:“完毕”、“我讲完了”。如果设置了特定词,并且在讲话过程中触发,那么会把当前讲话的内容送给LLM * Turn words at th end of speaking */ turnEndWords: string[] | undefined; /** * 判断用户说话是否结束模式,默认是普通模式,不使用AI进行语义判断是否说话结束 * Whether to enable market segmentation, enabled by default */ mode: AICallTurnDetectionMode; /** * 语义断句自定义等待时间,单位:毫秒,范围:[0, 10000];如果小于0则不会发给服务端(使用服务端默认值-1,AI自动判断合适的等待时间) * 注意:如果是AICallTurnDetectionMode.Normal模式,semanticWaitDuration字段无效 * Semantic segmentation waiting time, unit: milliseconds. If less than 0, the server default value of -1 will be used. */ semanticWaitDuration: number; /** * 仅在mode = Semantic模式下生效,优先级高于SemanticWaitDuration。控制 AI 检测到停顿后,启动回应的快慢程度,有以下取值: * "Low":耐心等待,最长等待时间6秒,减少被打断风险 * "Medium":平衡等待(最长等待时间4秒),适用大多数场景 * "High":快速响应(最长等待时间2秒),提升速度但可能增加误切风险 * 其他值(包括空值):走服务端默认配置 */ eagerness: string | undefined; fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } export declare enum AICallAgentType { /** * 纯语音 * Voice Call */ VoiceAgent = 0, /** * 数字人 * Avatar Call */ AvatarAgent = 1, /** * 视觉理解 * Vision Call */ VisionAgent = 2, /** * 视频通话 * Video Call */ VideoAgent = 3 } /** * VCR的基础检测配置 * VCR Base Detection Configuration */ declare class AICallAgentVcrBaseConfig { /** * 是否启用 * Whether to enable */ enable: boolean; constructor(enable?: boolean); /** * 初始化(通过toJSON方法生成的字典) * Initialization (via a dictionary generated by the toJSON method) */ fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } /** * VCR配置 * VCR configuration */ export declare class AICallAgentVcrConfig { private data; /** * VCR 的静止帧检测配置 */ stillFrameMotion?: AICallAgentVcrFrameMotionConfig; /** * VCR 的无效帧检测配置 */ invalidFrameMotion?: AICallAgentVcrFrameMotionConfig; /** * VCR 的实时画面人数检测配置 */ peopleCount?: AICallAgentVcrBaseConfig; /** * VCR 的电子设备检测配置 */ equipment?: AICallAgentVcrBaseConfig; /** * VCR 的头部动作检测配置 */ headMotion?: AICallAgentVcrBaseConfig; /** * VCR 的视线偏移检测配置 */ lookAway?: AICallAgentVcrBaseConfig; fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } /** * VCR的帧运动检测配置 * VCR Frame Motion Detection Configuration */ declare class AICallAgentVcrFrameMotionConfig extends AICallAgentVcrBaseConfig { /** * 多少毫秒后触发回调,默认3000ms */ callbackDelay: number; constructor(enable?: boolean, callbackDelay?: number); /** * 初始化(通过toJSON方法生成的字典) * Initialization (via a dictionary generated by the toJSON method) */ fromJSON(data: JSONObject): void; toJSON(): JSONObject; } /** * VCR检测Result */ export declare class AICallAgentVcrResult { data: JSONObject; stillFrameMotion?: FrameMotionResult; invalidFrameMotion?: FrameMotionResult; peopleCount?: PeopleCountResult; equipment?: EquipmentResult; headMotion?: HeadMotionResult; lookAway?: LookAwayResult; constructor(resultData: JSONObject); } export declare class AICallAgentVoiceprintConfig { /** * 当前断句是否使用声纹降噪识别 * Whether the current sentence uses voiceprint noise reduction recognition */ useVoiceprint: boolean; /** * 声纹Id,如果不为空表示当前通话开启声纹降噪能力,为空表示不启用声纹降噪能力 * Voiceprint ID. If not empty, it indicates that the current call enables voiceprint noise reduction capability. If empty, voiceprint noise reduction is not enabled. */ voiceprintId: string | undefined; fromJSON(data: JSONObject): void; /** * 转化为字典格式 * Convert to dictionary */ toJSON(): JSONObject; } export declare type AICallCameraConfig = { width?: number; height?: number; frameRate?: number; bitrate?: number; deviceId?: string; }; export declare class AICallChatSyncConfig { /** * 关联的chat智能体会话Id */ readonly sessionId: string; /** * 关联的chat智能体Id(必须同一账号同一区域上),如果设置了,那么在通话过程中会把通话记录同步到chat智能体上 */ readonly agentId: string; /** * 关联的chat智能体会话的用户Id */ readonly receiverId: string; /** * 初始化 * @param sessionId 关联的chat智能体会话Id * @param agentId 关联的chat智能体Id * @param receiverId 关联的chat智能体会话的用户Id */ constructor(sessionId: string, agentId: string, receiverId: string); /** * 生成Service请求格式的json字符串(不包含SessionId) * @returns JSON字符串,包含IMAIAgentId和ReceiverId */ getConfigString(): string; } export declare type AICallConfig = { agentId: string; agentType: AICallAgentType; agentUserId?: string; region: string; userId: string; /** * RTC Token of User */ userJoinToken: string; userData?: string; /** * @deprecated use agentConfig instead */ templateConfig?: AICallTemplateConfig; agentConfig?: AICallAgentConfig; chatSyncConfig?: AICallChatSyncConfig; }; export declare enum AICallConnectionStatus { /** 初始化完成 */ Init = 0, /** 网络连接断开 */ Disconnected = 1, /** 建立网络连接中 */ Connecting = 2, /** 网络已连接 */ Connected = 3, /** 重新建立网络连接中 */ Reconnecting = 4, /** 网络连接失败 */ Failed = 5 } export declare type AICallEngineConfig = { /** * 是否静音麦克风,默认 false * @deprecated use mute instead */ muteMicrophone?: boolean; /** *视觉理解模式默认是否关闭摄像头,默认 false * @deprecated use muteLocalCamera instead */ muteCamera?: boolean; /** * @deprecated * 是否开启按住说话,默认 false */ enablePushToTalk?: boolean; /** * 视觉理解模式 预览视频元素 */ previewElement?: string | HTMLVideoElement; /** * Agent 视频预览元素 */ agentElement?: string | HTMLVideoElement; /** * 视觉理解模式 摄像头配置 * @cameraConfig,width 宽度 * @cameraConfig,height 高度 * @cameraConfig,frameRate 帧率 * @cameraConfig,bitrate 码率 * @cameraConfig,deviceId 设备ID */ cameraConfig?: AICallCameraConfig; pushToTalkConfig?: AICallPushToTalkConfig; /** * 视觉理解模式 自定义摄像头 */ customCameraTrack?: MediaStreamVideoTrack; /** * 引擎配置 */ rtcEngineConfig?: { environment?: 'PRE' | 'PROD'; useAudioPlugin?: boolean; dumpAudioData?: boolean; useAIAEC?: boolean; enableSoft3ACheck?: boolean; useSysEchoCancellation?: boolean; useSysNoiseSuppression?: boolean; useSysAutoGainControl?: boolean; cameraContentHint?: string; }; /** * @deprecated * 模板配置 * 参考:https://help.aliyun.com/zh/ims/developer-reference/api-ice-2020-11-09-generateaiagentcall */ templateConfig?: AICallTemplateConfig; }; declare interface AICallEngineEvents { /** * 智能体状态改变 * @param state 当前智能体状态 * @deprecated use agentStateChanged instead */ agentStateChange: (newState: AICallAgentState) => void; /** * 智能体状态改变 * @param state 当前智能体状态 */ agentStateChanged: (newState: AICallAgentState) => void; /** * 智能体回答结果通知 * @param subtitle 智能体字幕信息 * @note subtitle.text 智能体字幕文本 * @note subtitle.sentenceId 智能体字幕语句 Id * @note subtitle.end 智能体字幕当前语句是否结束 */ agentSubtitleNotify: (subtitle: AICallSubtitleData) => void; /** * 用户语音识别结果通知 * @param subtitle 用户字幕信息 * @note subtitle.text 用户字幕文本 * @note subtitle.sentenceId 用户字幕语句 Id * @note subtitle.end 用户字幕当前语句是否结束 */ userSubtitleNotify: (subtitle: AICallSubtitleData, voiceprintResult: AICallVoiceprintResult) => void; /** * 智能体情绪结果通知 * @param emotion 情绪标签,例如:neutral\happy\angry\sad 等 * @param userAsrSentenceId 回答用户问题的句子ID */ agentEmotionNotify: (emotion: string, userAsrSentenceId: number) => void; /** * 当前通话的语音智能打断状态变更 * @param enable 是否启用 */ voiceInterruptChanged: (enable: boolean) => void; /** * 当前通话的音色发生了改变 * @param voiceId 当前音色Id */ voiceIdChanged: (voiceId: string) => void; /** * 当前通话对讲机模式变更 * @param enable 是否启用 */ pushToTalkChanged: (enable: boolean) => void; newRTCToken: (token: string) => void; authInfoWillExpire: () => void; authInfoExpired: () => void; /** * 音量变化通知 * @param uid 当前讲话人的uid * @param volume 音量[0-255] */ speakingVolumeChanged: (userId: string, volume: number) => void; /** * 音频订阅成功 * @param userId 订阅用户Id * @param audioElement 音频元素 */ audioSubscribed: (userId: string, audioElement?: HTMLAudioElement) => void; /** * 发生了错误 * @param code 错误码 */ errorOccurred: (code: number, msg: string) => void; /** * 通话开始(入会) */ callBegin: () => void; /** * 通话结束(离会) */ callEnd: () => void; /** * 通话智能体已启动 */ agentStarted: () => void; /** * 真人即将接管当前智能体 * @param takeoverUid 真人uid * @param takeoverMode 1:表示使用真人音色输出;0:表示使用智能体音色输出 */ humanTakeoverWillStart: (takeoverUid: string, mode: number) => void; /** * 真人接管已经接通 * @param takeoverUid 真人uid */ humanTakeoverConnected: (takeoverUid: string) => void; /** * 当前智能体即将离开(结束当前通话) * @param reason 原因:2001(闲时退出) , 2002(真人接管结束) 0(其他) * @param message 原因描述 */ agentWillLeave: (reason: number, message: string) => void; /** * 收到当前智能体发过来的自定义消息 * @param data 消息内容 */ receivedAgentCustomMessage: (data: { [key: string]: string | number | boolean | object | undefined | null; }) => void; /** * 当前Vision通话是否启用了自定义截帧模式 */ visionCustomCaptureChanged: (enable: boolean) => void; /** * 智能体当前讲话被打断 * @param reason 被打断的原因 */ speakingInterrupted: (reason: AICallSpeakingInterruptedReason) => void; /** * 当前通话中,LLM回答完毕 * @param text LLM输出的文本 * @param userAsrSentenceId 回答用户问题的句子ID * * LLM reply completed notification * @param text Text of the LLM's reply * @param userAsrSentenceId Sentence ID for answering the user's question */ llmReplyCompleted: (text: string, sentenceId: number) => void; /** * 智能体消息通道可用回调 * 需要等回调触发后,才可以给智能体发送消息 * * Agent data channel available * You need to wait for the callback to be triggered before you can send a message to the agent */ agentDataChannelAvailable: () => void; /** * 自动播放失败,需要引导用户点击页面来播放 * audo play failed, need to guide user to click the page to play */ autoPlayFailed: () => void; /** * 收到当前智能体发过来VCR结果 * @param result vcr结果 * * Received a VCR result from the current agent * @param result VCR Result */ receivedAgentVcrResult: (result: AICallAgentVcrResult) => void; latencyStats: (stats: LatencyStat) => void; connectionStatusChange: (status: AICallConnectionStatus, reason: number) => void; } export declare enum AICallErrorCode { /** * 成功 */ None = 0, /** * 操作无效 */ InvalidAction = -1, /** * 参数错误 */ InvalidParames = -2, /** * 网络错误 */ NetworkError = -3, /** * 内部错误 */ InternalError = -4, /** * 启动通话失败 */ BeginCallFailed = -10000, /** * 链接出现问题 */ ConnectionFailed = -10001, /** * 推流失败 */ PublishFailed = -10002, /** * 拉流失败 */ SubscribeFailed = -10003, /** * 通话认证过期 */ TokenExpired = -10004, /** * 同名登录导致通话无法进行 */ KickedByUserReplace = -10005, /** * 被系统踢出导致通话无法进行 */ KickedBySystem = -10006, /** * 频道被销毁导致通话无法进行 */ KickedByChannelTerminated = -10007, /** * 本地设备问题导致无法进行 */ LocalDeviceException = -10008, /** * 智能体离开频道了(智能体结束通话) */ AgentLeaveChannel = -10101, /** * 智能体拉流失败了 */ AgentPullFailed = -10102, /** * 智能体ASR失败 */ AgentASRFailed = -10103, /** * 数字智能体服务启动失败 */ AvatarServiceFailed = -10201, /** * 数字智能体超出并发路数 */ AvatarRoutesExhausted = -10202, /** * 发起通话,超出每天免费体验的额度 */ AgentSubscriptionRequired = -10203, /** * 智能体未能找到(智能体ID不存在) */ AgentNotFound = -10204, /** * 文本消息发送失败 */ ChatTextMessageSendFailed = -10301, /** * 文本消息接收失败 */ ChatTextMessageReceiveFailed = -10302, /** * 语音消息发送失败 */ ChatVoiceMessageSendFailed = -10303, /** * 语音消息发送失败 */ ChatVoiceMessageReceiveFailed = -10304, /** * 语音消息录制失败 */ ChatVoiceRecordFailed = -10305, /** * 聊天记录未能找到 */ ChatLogNotFound = -10306, /** * 附件的还在上传中,需要上传完成后,才能发送消息 * Attachment is uploading */ ChatAttachmentUploading = -10332, /** * 附件的发送器还在上传中 * Attachment's sender is sending */ ChatAttachmentSenderSending = -10333, /** * 当前消息的附件发送器已经被释放,无法再发送 * Attachment‘s sender has destroied */ ChatAttachmentSenderDestroy = -10334, /** * 未知错误 */ UnknowError = -40000 } export declare interface AICallMessageReceiveData { type: AICallMessageType; data: JSONObject; seqId: number; senderId?: string; receiverId?: string; } export declare interface AICallMessageSendData { type: AICallMessageType; data?: JSONObject; senderId?: string; receiverId?: string; } export declare enum AICallMessageType { None = 0, AgentErrorOccurs = 1000, AgentStateChanged = 1001, AgentSubtitleNotify = 1002, UserSubtitleNotify = 1003, VoiceInterruptChanged = 1004, VoiceIdChanged = 1005, RTCTokenResponsed = 1006, PushToTalkChanged = 1007, VoiceprintUsingChanged = 1008, VoiceprintClearSucceed = 1009, AgentWillLeave = 1010, CustomMessageReceived = 1011, HumanTakeoverWillStart = 1012, HumanTakeoverConnected = 1013, AgentEmotionNotify = 1014, VisionCustomCaptureChanged = 1015, SpeakingInterruptedReason = 1016, LLMReplyCompleted = 1017, VcrResultReceived = 1018, StopAgent = 1100, InterruptSpeaking = 1101, EnableVoiceInterrupt = 1102, SwitchVoiceId = 1103, RequestRTCToken = 1104, EnablePushToTalk = 1105, StartPushToTalk = 1106, FinishPushToTalk = 1107, CancelPushToTalk = 1108, UseVoiceprint = 1109, ClearVoiceprint = 1110, SendTextToAgent = 1111, SendCustomMessageToServer = 1112, StartVisionCustomCapture = 1113, StopVisionCustomCapture = 1114, UpdateLlmSystemPrompt = 1115, UpdateVcrConfig = 1116, UpdateBailianAppParams = 1117, UpdateAsrMaxSilence = 1118, QueryAgentState = 1119, UpdateTtsSpeechRate = 1120 } export declare type AICallPushToTalkConfig = { /** * 停止按住说话时是否关闭麦克风,默认 false */ closeMicrophoneOnRelease?: boolean; /** * 停止按住说话时是否关闭摄像头,默认 false */ pushVideoOnRelease?: boolean; }; export declare class AICallSendTextToAgentRequest { text: string; constructor(text: string); } /** * 智能体当前讲话被打断原因 */ export declare enum AICallSpeakingInterruptedReason { /** * 未知原因 * Interrupted due to unknown */ unknown = 0, /** * 因为识别到特定词 * Interrupted due to recognition of specific words */ byWords = 1, /** * 因为语音打断 * Interrupted due to voice input */ byVoice = 2, /** * 因为调用了API:interruptSpeaking * Interrupted due to interruptSpeaking API */ byInterruptSpeaking = 3, /** * 因为主动语音播报打断 * Interrupted due to speech broadcast */ bySpeechBroadCast = 4, /** * 因为主动LLM查询打断 * Interrupted due to llm query */ byLlmQuery = 5 } export declare enum AICallState { None = 0,// 初始化 Connecting = 1,// 接通中 Connected = 2,// 通话中 Over = 3,// 通话结束 Error = 4 } export declare interface AICallSubtitleData { text: string; sentenceId: number; end: boolean; } export declare class AICallTemplateConfig { /** * 智能体欢迎语,为空表示使用智能体配置值 */ agentGreeting?: string; /** * 用户未入会,智能体超时关闭任务的时间,小于0则使用服务端默认值60s */ userOnlineTimeout: number; /** * 用户退会后,智能体超时关闭任务的时间,小于0则使用服务端默认值5s */ userOfflineTimeout: number; /** * 工作流覆盖参数 */ workflowOverrideParams?: JSONObject; /** * 百炼应用中心参数 */ bailianAppParams?: JSONObject; /** * 语音断句检测阈值,静音时长超过该阈值会被认为断句,参数范围 200ms~1200ms,小于0则使用服务端默认值400ms */ asrMaxSilence: number; /** * 智能体说话的音量,范围为 0~400,输出音量=工作流中的语音输出音量 * volume/100,小于0则使用服务端默认值100 */ volume: number; /** * 设置AIVad的灵敏度参数,通过该参数的设定可以抗人声干扰。取值范围[0-11],0表示关闭AIVad,服务端默认值是0,客户端默认开启AIVad且值为11 * Set the sensitivity parameter for AIVad. By setting this parameter, resistance to human voice interference can be achieved. The valid range is [0-10]. A value of 0 means AIVad is disabled. The server default value is 0, but the client enables AIVad by default with a value of 11. */ vadLevel: number; /** * 是否开启智能打断 */ enableVoiceInterrupt: boolean; /** * 智能体讲话音色Id,为空表示使用智能体配置值 */ agentVoiceId?: string; /** * 语音通话的智能体头像的头像链接 */ avatarUrl?: string; /** * 智能断句开关,开启智能断句后,用户说话的发生断句会智能合并成一句 */ enableIntelligentSegment: boolean; /** * 智能体闲时的最大等待时间(单位:秒),超时智能体自动下线,设置为-1表示闲时不退出,小于0则使用服务端默认值600s */ agentMaxIdleTime: number; /** * 是否开启对讲机模式 */ enablePushToTalk: boolean; /** * 数字人模型Id,为空表示使用智能体配置值 */ agentAvatarId?: string; /** * 是否优雅下线 * 优雅下线:当智能体被停止的时候,播报完当前说的话再停止,最多播报 10 秒 */ agentGracefulShutdown: boolean; /** * asr语种Id,为空表示使用智能体配置值 */ asrLanguageId?: string; /** * 用户在通话启动前的指令,用于智能体在通话启动后立即响应,例如:“今天天气怎么样?” */ wakeUpQuery?: string; /** * llm的系统提示词,例如:“你是一位友好且乐于助人的助手,专注于为用户提供准确的信息和建议。” * 注意:不支持LLM节点为百炼工作流类型,返回成功不代表生效 */ llmSystemPrompt?: string; /** * llm/mllm 历史对话上下文最大保留轮次,小于0则使用服务端默认值 10 次 * The maximum number of rounds for retaining historical conversation context in llm/mllm. If it is less than 0, the server default value of 10 rounds will be used. */ llmHistoryLimit: number; /** * 触发对话打断的特定词汇或短语,例如:“打断一下”、“我知道了” * Specific words or phrases that trigger conversation interruption, e.g., "Interrupt," "I know." */ interruptWords?: string[]; /** * ASR热词列表 * Hot words for ASR */ asrHotWords?: string[]; /** * VCR配置 * VCR configuration */ vcrConfig?: AICallAgentVcrConfig; getJsonString(agentType: AICallAgentType): string; static getTemplateConfigKey(agentType: AICallAgentType): string; static fromJsonString(agentType: AICallAgentType, jsonString: string): AICallTemplateConfig; } export declare enum AICallTurnDetectionMode { /** * 普通模式,不使用AI进行语义判断是否说话结束,使用ASR静默时间判断 * Normal mode: AI is not used to determine whether the speech has ended. * ASR is used to determine the silence time. */ Normal = 0, /** * Semantic,用AI根据上下文语义,判断是否说话结束 * Semantic, using AI to determine whether a speech has ended based on contextual semantics */ Semantic = 1 } export declare interface AICallVisionCustomCaptureRequest { /** * 请求多模态大模型时的文本参数 */ text?: string; /** * 为true时表示单次截帧,截帧后立即退出自定义截帧状态; * 为false时表示持续截帧,达到duration后会自动退出自定义截帧状态 */ isSingle?: boolean; /** * 截帧间隔时长 * 默认 5 秒一次 */ eachDuration?: number; /** * 每次截帧的图片数量 * 默认 2 */ num?: number; /** * 持续截帧的时长(秒),如果isSingle=true,则忽略,否则达到该设定时长时会结束自定义截帧状态 */ duration?: number; /** * json字符串,自定义业务信息,跟随text和图片帧一同传给大模型,用于企业做业务处理 */ userData?: string; } export declare enum AICallVoiceprintResult { /** * 没开启声纹降噪Vad,且关闭AIVad * Voiceprint noise reduction Vad is not enabled, and AIVad is disabled */ Off = 0, /** * 已开启声纹降噪Vad,但还完成声纹注册 * Voiceprint noise reduction Vad is enabled, but voiceprint registration is not yet completed */ Unregister = 1, /** * 已开启声纹降噪Vad,识别到主讲人 * Voiceprint noise reduction Vad is enabled, and the speaker has been detected */ DetectedSpeaker = 2, /** * 已开启声纹降噪Vad,没识别到主讲人 * Voiceprint noise reduction Vad is enabled, but the speaker has not been detected */ UndetectedSpeaker = 3, /** * 开启AIVad,识别到主讲人 * AIVad is enabled, and the speaker has been detected */ DetectedSpeakerWithAIVad = 4, /** * 开启AIVad,但没识别到主讲人 * AIVad is enabled, but the speaker has not been detected */ UndetectedSpeakerWithAIVad = 5, /** * 未知 * Unknown */ Unknown = 100 } /** * Chat智能体信息 */ export declare class AIChatAgentInfo { /** * 智能体Id */ readonly agentId: string; /** * 智能体服务所在的区域,默认使用上海区域 */ region: string; /** * 初始化 */ constructor(agentId: string, region?: string); } /** * Chat智能体响应状态 */ export declare enum AIChatAgentResponseState { /** * 聆听中 */ Listening = 1, /** * 思考中 */ Thinking = 2, /** * 回复中 */ Replying = 3 } /** * 分享智能体配置信息 */ export declare class AIChatAgentShareConfig { /** * 智能体分享ID */ readonly shareId?: string; /** * 过期时间 */ readonly expireTime?: Date; /** * 服务所在区域 */ readonly region?: string; /** * 初始化 */ constructor(shareId?: string, expireTime?: Date, region?: string); /** * 初始化 */ static fromData(data?: JSONObject): AIChatAgentShareConfig | undefined; } export declare enum AIChatAgentType { MessageChat = 11 } /** * 附件Model * Attachment model */ export declare class AIChatAttachment extends default_2 { /** * 附件ID * Attachment ID */ id: string; /** * 附件类型 * Attachment type */ type: AIChatAttachmentType; /** * 附件名称 * Attachment name */ name: string; /** * 附件文件路径(可能是本地路径 或 网络路径) * Attachment path */ path: string; /** * 附件的额外信息 * Attachment extra info */ extra?: string; /** * 附件的状态 * Attachment state */ private _state; /** * 附件的上传进度,百分比进度:0.00-100.00 * Attachment progress */ private _progress; /** * 附件文件或文件地址(小程序) * Attachment file */ file?: File; /** * 附件的响应 * Attachment response */ private _response?; /** * 初始化 * Initialization */ constructor(attachmentId?: string, type?: AIChatAttachmentType, name?: string, path?: string, extra?: string, state?: AIChatAttachmentState, progress?: number, file?: File); static createImageAttachment(file: File): AIChatAttachment; static createAudioAttachment(file: File): AIChatAttachment; static createVideoAttachment(file: File): AIChatAttachment; static createFileAttachment(file: File): AIChatAttachment; set progress(progress: number); get progress(): number; set state(state: AIChatAttachmentState); get state(): AIChatAttachmentState; set response(response: ImAttachmentRes); get response(): ImAttachmentRes | undefined; /** * 初始化(通过通话记录返回的数据) * Initialization (via data returned from call records) */ static fromDialogueData(dialogueData: JSONObject): AIChatAttachment; /** * 初始化(通过toData方法生成的字典) * Initialization (via a dictionary generated by the toData method) */ static fromData(data?: JSONObject): AIChatAttachment; /** * 生成数据字典 * Generate a data dictionary */ toJSON(): JSONObject; } declare interface AIChatAttachmentEvents { /** * 附件上传进度 * @param attachment 附件 * @param progress 进度,百分比0-100 * * Attachment upload progress * @param attachment * @param progress */ progress: (progress: number) => void; /** * 附件状态改变 * Attachment state change * @param state 附件状态 * * Attachment state change * @param state */ stateChange: (state: AIChatAttachmentState) => void; } /** * 附件状态 * Attachment State */ export declare enum AIChatAttachmentState { /** * 初始状态 * Init */ Init = 0, /** * 上传中 * Adding */ Uploading = 1, /** * 上传成功 * Success */ Success = 2, /** * 上传失败 * Failed */ Failed = 3 } /** * 附件类型 * Attachment type */ declare enum AIChatAttachmentType { /** * 未知 * None */ None = 0, /** * 图片 * Image */ Image = 1, /** * 音频文件(暂不支持) * Audio */ Audio = 2, /** * 视频文件(暂不支持) * Video */ Video = 3, /** * 其他文件类型 * Other */ Other = 4 } export declare class AIChatAttachmentUploader extends default_2 { attachmentList: AIChatAttachment[]; static getInputModalities(attachmentList?: AIChatAttachment[]): string[]; constructor(); get attachmentManager(): AliVCIMAttachmentManager | undefined; get allUploadSuccess(): boolean; get manager(): AliVCIMAttachmentManager | undefined; addAttachment(attachment: AIChatAttachment): Promise; removeAttachment(attachmentId: string): Promise; destroy(shouldDelete?: boolean): void; private addAndUploadIMAttachment; private stopAndRemoveIMAttachment; } /** * 附件上传回调 * Attachment upload callback */ declare interface AIChatAttachmentUploadEvents { /** * 上传成功回调 * @param attachment 触发回调的附件 * * Upload success callback * @param attachment */ uploadSuccess?(attachment: AIChatAttachment): void; /** * 上传失败回调 * @param attachment 触发回调的附件 * @param error 错误信息 * * Failure Callback * @param attachment * @param error Error information */ uploadFailure?(attachment: AIChatAttachment, error: Error): void; } /** * Chat智能体链接Token */ export declare class AIChatAuthToken { /** * 应用id */ appId: string; /** * 应用签名 */ appSign: string; /** * 鉴权token */ token: string; /** * 过期时间戳 */ timestamp: number; /** * 角色,为admin时,表示该用户可以调用管控IM接口,可为空 */ role?: string; /** * "AK-随机串", 最长64字节, 仅限A-Z,a-z,0-9及"_",可为空 */ nonce?: string; /** * 初始化 */ constructor(appId: string, appSign: string, token: string, timestamp: number, role?: string, nonce?: string); /** * 初始化(key规范为:小写+下划线分割) */ static fromData(data: JSONObject | null): AIChatAuthToken; /** * 当前token是否有效 */ isValid(): boolean; } export declare class AIChatEngine extends default_2 { private _state; userInfo?: AIChatUserInfo; agentInfo?: AIChatAgentInfo; agentResponeState: AIChatAgentResponseState; sessionId?: string; templateConfig?: AIChatTemplateConfig; userData?: JSONObject; private audioCtx; voiceIdList: string[]; private curRound?; get audioContext(): AudioContext; get currentRecordStream(): MediaStream | undefined; set state(state: AIChatEngineState); get state(): AIChatEngineState; get round(): AIChatRound | undefined; private onRequestAuthToken; private addIMListener; private removeIMListener; /** * 开始聊天会话 * @param userInfo 当前开始会话的用户信息 * @param agentInfo 当前开始会话的智能体消息 * @param sessionId 当前会话的唯一标识,如果为空,则使用userId+agentId作为唯一标识 */ startChat(userInfo: AIChatUserInfo, agentInfo: AIChatAgentInfo, sessionId?: string): Promise; /** * 结束聊天会话 * @param needLogout 当前用户实时要退出链接(设置为fals会在下次startChat较少链接耗时,当你后续都不再进行聊天时,可以设置为true) */ endChat(needLogout: boolean): Promise; /** * 向Chat智能体发送消息,发送前会打断当前的回复(如果在思考&回复中) * @param request 要发送的消息内容 * @param completed 发送结果回调 */ sendMessage(userMsg: AIChatMessage, uploader?: AIChatAttachmentUploader): Promise; /** * 打断当前的消息回复 */ interruptAgentResponse(): void; /** * 删除消息 * @param dialogueId 消息Id * @param completed 结果回调 */ deleteMessage(dialogueIdOrMessage: string | AIChatMessage): Promise; queryMessageList(request: AIChatMessageListRequest): Promise; private generateUUID; /** * 开始播放消息内容 * @param message 要播放的消息 * @param voiceId 播放的音色,不为空时使用该音色进行播报,并会设置templateConfig里的voiceId * @param completed 结果回调 */ startPlayMessage(message: AIChatMessage, voiceId?: string): Promise; /** * 结束消息的播放 */ stopPlayMessage(needReject?: boolean): void; /** * 是否播放中 * @param dialogueId 消息Id */ isPlayingMessage(dialogueId: string): boolean; /** * 开始按键语音,按下去时开始录音并实时发送,需确保已经开启录音权限 * @param request 要发送的消息内容 * @return true表示成功 */ startPushVoiceMessage(uploader?: AIChatAttachmentUploader): Promise; /** * 结束按键语音,松开后时调用,表示当前语音消息发送完毕,等待智能体返回结果 */ finishPushVoiceMessage(): Promise; /** * 取消按键语音,松开后时调用,表示取消当前语音消息的发送 */ cancelPushVoiceMessage(): void; /** * 创建附件上传器 * Create attachment uploader */ createAttachmentUploader(): Promise; destroy(): void; static parseShareAgentChat(shareInfo: string): AIChatAgentShareConfig | undefined; generateShareAgentChat(shareConfig: AIChatAgentShareConfig, userId: string): Promise<(string | AIChatAgentInfo | AIChatAuthToken)[]>; private handleErrorData; private onTextQueryResult; private onTTSQueryResult; private onSpeechQueryResult; private onSpeechQueryError; private onSendSpeechQueryData; private onReceivedMessageData; private onReceivedStreamMessageError; private onReceivedTextResponse; private checkAgentStateChanged; private onReceivedTTSResponse; private onReceivedSpeechTranscription; private fetchVoiceList; private loginIM; private sendIMStreamTextMessage; private createIMStreamSender; private sendIMStreamMessage; private rejectStreamMessage; private sendIMControllerMessage; private processAgentResponseMsgError; private processAgentResponseMsgByInterruption; private processAgentResponseMsg; private updateAgentResponseRound; private updateAgentResponseMsg; } /** * Chat智能体引擎回调 */ export declare interface AIChatEngineEvents { /** * 发生了错误 * @param error 错误码 * @param requestId 消息请求Id,当不为null时,该错误属于消息错误 */ errorOccurs: (error: AIChatError, requestId?: string) => void; /** * 请求AuthToken,当ChatEngine需要执行IM登录或者IM登录AuthToken后过期时会触发 * @param userId 用户id */ requestAuthToken: (userId: string, responseBlock: (authToken?: AIChatAuthToken, error?: AIChatError) => void) => void; /** * Chat智能体链接改变 * @param state 当前Chat智能体链接状态 */ engineStateChange: (state: AIChatEngineState) => void; /** * 收到Chat智能体的回复消息 * 1、智能体新的回复或者当前回复过程中消息的状态发生了变化都会触发该回调 * 2、通话类型的智能体的字幕(用户asr字幕、智能体回复)也会通过该回调返回 * @param message 消息对象 */ receivedMessage: (message: AIChatMessage) => void; /** * 收到自定义消息 * @param text 消息内容 */ receivedCustomMessage: (text: string) => void; /** * 用户发送的消息需要更新 * @param message 消息对象 */ userMessageUpdated: (message: AIChatMessage) => void; /** * 收到Chat智能体响应改变 * @param state 响应状态 * @param requestId 当前的消息会话 */ agentResponeStateChange: (state: AIChatAgentResponseState, requestId?: string) => void; /** * 收到Chat智能体的消息播放状态改变 * @param message 消息对象 * @param state 播放状态 */ messagePlayStateChange: (message: AIChatMessage, state: AIChatMessagePlayState) => void; /** * 收到Chat智能体的语音识别结果 * @param message 消息对象 */ receivedSpeechTranscription: (message: AIChatMessage) => void; /** * 收到Chat智能体的音色列表更新 * @param voiceIdList 音色列表 */ voiceListUpdated: (voiceIdList: string[]) => void; } /** * Chat智能体链接状态 */ export declare enum AIChatEngineState { /** * 未链接 */ Init = 0, /** * 链接中 */ Connecting = 1, /** * 已连接 */ Connected = 2, /** * 链接断开 */ Disconnect = 3 } export declare class AIChatError extends Error { code?: number; constructor(message: string, code?: number); } export declare type AIChatErrorCode = AICallErrorCode; /** * Chat智能体消息定义 */ export declare class AIChatMessage { /** * 消息ID */ dialogueId: string; /** * 当前dialogueID是否回答结束,表示一次完整的回答。如果不存在子消息(nodeId为空),则与isEnd值相同。 * Whther the current dialogue is end */ isDialogueEnd: boolean; /** * 如果当前是子消息,则用于标记子消息信息 * Node ID */ nodeId?: string; /** * 对话回合ID */ requestId: string; /** * 消息状态 */ messageState: AIChatMessageState; /** * 消息类型 */ messageType: AIChatMessageType; /** * 原始文本内容 */ text: string; isEnd: boolean; /** * 发送方用户标识:用户 或 智能体 */ senderId?: string; /** * 消息发送时间 */ sendTime?: number; /** * 消息发送源:chat(消息对话智能体) 或 call(通话智能体) */ source?: string; /** * 消息类型,不同场景下的值不同 * 通话场景: * 1. greeting 欢迎语 * 2. normal 语音回复 * 3. speech 主动播报 * 消息对话场景: * 1. normal 文本回复 * 2. announcement 主动推送文字 * 3. custom 自定义消息 */ sourceType?: string; /** * 推理过程中的内容 */ reasoningText?: string; /** * 推理是否结束 */ isReasoningEnd: boolean; attachmentList: AIChatAttachment[]; /** * 用户自定义扩展信息 * User extend info */ extend?: string; /** * 初始化 */ constructor(dialogueId: string, requestId: string, state: AIChatMessageState, type: AIChatMessageType, sendTime: number | undefined, text: string, senderId: string | undefined, isEnd: boolean, source?: string, sourceType?: string, attachmentList?: AIChatAttachment[], isDialogueEnd?: boolean, nodeId?: string, extend?: string); /** * 初始化 */ static fromSendText(sendText: string, attachmentList?: AIChatAttachment[], requestId?: string, state?: AIChatMessageState): AIChatMessage; /** * 初始化 */ static fromRequestId(requestId: string, attachmentList?: AIChatAttachment[]): AIChatMessage; /** * 初始化 */ static fromData(data?: JSONObject): AIChatMessage; /** * 初始化(通过通话记录返回的数据) */ static fromDialogueData(dialogueData: JSONObject | undefined, userId: string, agentId: string): AIChatMessage; /** * 初始化 */ static fromReceivedData(receivedData?: JSONObject): AIChatMessage; toData(): JSONObject; } export declare enum AIChatMessageDataType { TextQuery = "TextQuery", TextResponse = "TextResponse", TTSQuery = "TTSQuery", TTSResponse = "TTSResponse", SpeechQuery = "SpeechQuery", SpeechTranscription = "SpeechTranscription", DeleteAIAgentDialogue = "DeleteAIAgentDialogue", ListAIAgentDialogues = "ListAIAgentDialogues", DescribeAIAgent = "DescribeAIAgent", Error = "Error" } /** * Chat智能体消息列表请求 */ export declare interface AIChatMessageListRequest { /** * 开始Unix时间戳(不含),单位为秒 */ startTime: number; /** * 结束Unix时间戳(包含),单位为秒 */ endTime: number; /** * 分页数 */ pageNumber: number; /** * 分页大小,最大100,默认20 */ pageSize: number; /** * 是否倒序 */ isDesc: boolean; } /** * Chat智能体消息播放状态 */ export declare enum AIChatMessagePlayState { /** * 初始状态 */ Init = 0, /** * 播放中 */ Playing = 1, /** * 播放已停止 */ Stopped = 2, /** * 播放出错了 */ Failed = 3 } /** * Chat智能体消息状态 */ export declare enum AIChatMessageState { /** * 初始状态 */ Init = 1, /** * 传输中(发送中/接收中) */ Transfering = 2, /** * 输出中 */ Printing = 3, /** * 打断了 */ Interrupted = 4, /** * 结束了 */ Finished = 5, /** * 出错了 */ Failed = 6 } /** * Chat智能体消息类型 */ export declare enum AIChatMessageType { /** * 文本类型 */ Text = 0, /** * 语音类型 */ Voice = 1 } declare class AIChatRound extends default_2 { requestId: string; _roundType: string; userMsg?: AIChatMessage; agentResponseMsg?: AIChatMessage; playingMessage?: AIChatMessage; agentResponseSubMsgList: AIChatMessage[]; streamPlayer?: AIChatStreamPlayer; streamRecorder?: AIChatStreamRecorder; imSender?: ImStreamMessageSender; constructor(textMsg: AIChatMessage, agentResponseMsg: AIChatMessage); constructor(requestId: string, playingMessage: AIChatMessage); constructor(voiceMsg: AIChatMessage, asrSender: unknown); private _imMessageId?; set imMessageId(value: string); get imMessageId(): string | undefined; get roundType(): string; startPlay(sampleRate: number, channels: number, bitDepth: number, audioCtx?: AudioContext): void; pushPlayData(data: Uint8Array): void; finishPushPlayData(isFailed: boolean): void; isPlaying(dialogueId: string): boolean; stopPlay(): void; startRecord(): Promise; stopRecord(): Promise; onStreamRecorderCapturingBuffer(data: Uint8Array): void; } declare interface AIChatRoundEvents { streamPlayerStateChanged(state: AIChatStreamState): void; recordingBlock(data?: Uint8Array, info?: JSONObject): void; } /** * Chat智能体发送消息请求 */ export declare interface AIChatSendMessageRequest { /** * 消息类型 */ messageType: AIChatMessageType; /** * 原始文本内容,messageType为文本类型时,不能为空 */ text: string; /** * 消息会话ID */ requestId?: string; } /** * Chat流数据类型 */ declare enum AIChatStreamDataType { /** * 16位整数 */ Int16 = 0, /** * 32位浮点数 */ Float32 = 1 } /** * Chat流播放器 */ declare class AIChatStreamPlayer extends default_2 { /** * 采样率 */ readonly sampleRate: number; /** * 通道数 */ readonly channelCount: number; /** * 数据类型 */ readonly dataType: AIChatStreamDataType; /** * 当前状态 */ get state(): AIChatStreamState; private set state(value); /** * 当前时间 */ get currentTime(): number; /** * 持续时间 */ get duration(): number | null; private _state; private _currentTime; private _duration; private audioContext?; private sourceNode?; private processor?; private isPushEnd; private isPushFailed; constructor(audioCtx: AudioContext, sampleRate?: number, channelCount?: number, dataType?: AIChatStreamDataType); /** * 推送播放数据 * @param data 数据 * @param isEnd 是否结束 * @param isFailed 是否失败 */ pushPlayData(data: Uint8Array | null, isEnd?: boolean, isFailed?: boolean): Promise; /** * 停止播放 * @param reason 原因:0(主动stop) 1(播放结束) 2(播放失败) */ stop(reason?: number): void; } /** * Chat流播放器代理 */ declare interface AIChatStreamPlayerEvents { /** * 播放器状态改变 * @param state 当前播放器状态 */ streamPlayerStateChanged: (state: AIChatStreamState) => void; } /** * Chat流录音器 */ declare class AIChatStreamRecorder extends default_2 { /** * 采样率 */ readonly sampleRate: number; /** * 通道数 */ readonly channelCount: number; /** * 位深度 */ readonly bitDepth: number; /** * 是否 dump 文件 */ dumpFile: boolean; private isRecording; private audioContext?; private stream?; private sourceNode?; private workletNode?; get audioCtx(): AudioContext | undefined; get currentStream(): MediaStream | undefined; get currentSourceNode(): MediaStreamAudioSourceNode | undefined; /** * 初始化 * @param sampleRate 采样率,默认16000 * @param channelCount 通道数,默认1 * @param bitDepth 位深度,默认16 */ constructor(sampleRate?: number, channelCount?: number, bitDepth?: number); /** * 开始录音 * @returns 是否成功开始录音 */ start(): Promise; /** * 结束录音 */ finish(): Promise; } /** * Chat流录音器代理 */ declare interface AIChatStreamRecorderEvents { /** * 录音器捕获缓冲区 * @param data 数据 */ streamRecorderCapturingBuffer: (data: Uint8Array) => void; } /** * Chat流播放状态 */ declare enum AIChatStreamState { /** * 初始化 */ Init = 0, /** * 播放中 */ Playing = 1, /** * 停止 */ Stopped = 2, /** * 失败 */ Failed = 3, /** * 暂停 */ Ended = 4 } export declare class AIChatTemplateConfig { /** * 百炼应用中心参数 */ bailianAppParams?: JSONObject; /** * 智能体讲话音色Id,为空表示使用智能体配置值 */ agentVoiceId?: string; static fromJsonString(jsonString: string): AIChatTemplateConfig; /** * 将对象转换为 JSON 字符串 * @returns JSON 字符串 */ getJsonString(): string; } /** * Chat用户信息 */ export declare class AIChatUserInfo { /** * 用户Id */ readonly userId: string; /** * 设备id */ deviceId?: string; /** * 初始化 */ constructor(userId: string, deviceId?: string); } declare class AliVCIMAttachmentManager { private static instance; private wasmIns; private attachmentManager; private uploader; constructor(wasmIns: any, wasmInterface: any); static getInstance(wasmIns: any, wasmInterface: any): AliVCIMAttachmentManager; getAttachmentReq(attachmentReq: ImAttachmentReq): Promise; /** * 上传附件 * @param {string} reqId * @param {ImAttachmentReq} attachmentReq * @returns {ImAttachmentRes} */ uploadAttachment(reqId: string, attachmentReq: ImAttachmentReq): Promise; /** * 取消上传附件 * @param {string} reqId * @returns */ cancelAttachmentUpload(reqId: string): Promise; /** * 删除已上传附件 * @param {ImAttachmentRes} attachment * @returns */ deleteAttachment(attachment: ImAttachmentRes): Promise; destroy(): void; } declare class ARTCAICallEngine extends default_2 { static isSupported: (direction?: "sendrecv" | "recvonly" | "sendonly") => Promise; static getMicrophoneList: () => Promise; static getCameraList: () => Promise; static parseShareAgentCall(shareConfigToken: string): AICallAgentShareConfig; private audioProcessPlugin?; private _callState; private _rtcEngine?; private _dcReady; private _seqId; private _userId?; private _config; private _previewView?; private _agentType?; private _agentInfo?; private _agentState; private _avatarUserId?; private _agentView?; private _takeoverUid?; private _takeoverMode?; private _enablePushToTalk; private _isStartPushToTalk; constructor(config?: AICallEngineConfig); get userId(): string | undefined; get agentInfo(): AICallAgentInfo | undefined; get agentState(): AICallAgentState; get hasLocalVideo(): boolean; get hasAgentVideo(): boolean; get getRTCInstance(): default_3 | undefined; private onDataChannelMsg; private handleLocalUserVolume; /** * 初始化引擎 * @note 您可以在处理化过程中同时请求 GenerateAIAgentCall 接口来降低通话启动时长 * @param config 配置 */ init(agentType: AICallAgentType, config?: AICallEngineConfig): Promise; /** * 呼叫 * @param userId 用户 id * @param agentInfo 智能体信息 * @param agentType (可选)智能体类型 * @param config (可选)配置 */ call(userId: string, _agentInfo: AICallAgentInfo, agentTypeOrConfig?: AICallAgentType | AICallEngineConfig, _config?: AICallEngineConfig): Promise; callWithConfig(config: AICallConfig): Promise; private startPlayAgentVideo; /** * 设置 Agent 视频视图 * set view for agent * @param view 视频视图,如果不传会停止播放 element or id, if not set, will stop play */ setAgentView(view?: HTMLVideoElement | string): void; /** * 设置本地预览视图 * set view for local camera * @param view 视频视图,如果不传会停止预览 element or id, if not set, will stop preview */ setLocalView(view?: HTMLVideoElement | string): void; /** * 开始预览 * start preview * @deprecated use setLocalView instead * @param view */ startPreview(view: HTMLVideoElement | string): void; /** * 停止预览 * stop preview * @deprecated use setLocalView instead */ stopPreview(): void; /** * 挂断 */ handup(): Promise; /** * 设置智能体渲染视图,及缩放模式 */ destroy(): Promise; /** * 静音 */ mute(mute: boolean): Promise; switchMicrophone(deviceId: string): Promise; /** * 停止/恢复智能体音频流的播放 * @param mute 是否静音 * @return 是否成功 */ /**** * Mute/unmute agent audio playing * @param mute Whether to mute * @return Whether it is successful */ muteAgentAudioPlaying(mute: boolean): Promise; muteLocalCamera(mute: boolean): Promise; switchCamera(deviceId?: string): Promise; updateToken(token: string): Promise; stopAgent(): Promise; /** * 主动打断讲话 */ interruptSpeaking(): Promise; /** * 开启/关闭智能打断 */ enableVoiceInterrupt(enable: boolean): Promise; /** * 设置语音识别 */ switchVoiceId(voiceId: string): Promise; requestRTCToken(): Promise; /** * 更新TTS播报语速,支持所有TTS类型,取值范围为[0.5, 2.0] * @param rate 语速 * @return 是否发送成功,不代表是否生效 * * * Update TTS speech rate, supported by all TTS types, value range is [0.5, 2.0]. * @param rate rate * @return Whether successful */ /** * 更新语音断句检测阈值,静音时长超过该阈值会被认为断句,参数范围 200ms~1200ms * Update voice sentence detection threshold. Silence exceeding this threshold will be considered as a sentence break. The range is 200ms to 1200ms. * @param duration 时长(毫秒) * @returns 是否发送成功,不代表是否生效 */ updateAsrMaxSilence(duration: number): boolean; /** * 更新TTS播报语速,支持所有TTS类型,取值范围为[0.5, 2.0] * Update TTS speech rate, supported by all TTS types, value range is [0.5, 2.0]. * @param rate rate * @return Whether successful */ updateTtsSpeechRate(rate: number): boolean; enablePushToTalk(enable: boolean): boolean; private rtcStartTalk; startPushToTalk(): boolean; private rtcStopTalk; finishPushToTalk(): boolean; cancelPushToTalk(): boolean; sendTextToAgent(req: AICallSendTextToAgentRequest | string): boolean; /** * 查询当前的智能体状态,发送成功后,会触发onAgentStateChanged事件 * @return 是否发送成功 * * * Query current agent state, if successful will raise onAgentStateChanged event. * @return Whether successful */ queryCurrentAgentState(): boolean; /** * 开启视觉理解自定义摄像头 * @param req 开启视觉理解自定义摄像头请求 * @return 是否发送成功 * * * Start vision custom capture * @param req Start vision custom capture request * @return Whether successful */ /** * 给Server发送自定义消息 */ sendCustomMessageToServer(message: string): boolean; /** * 更新llm的系统提示词,例如:“你是一位友好且乐于助人的助手,专注于为用户提供准确的信息和建议。” * 注意:不支持LLM节点为百炼工作流类型 */ updateLlmSystemPrompt(prompt: string): boolean; /** * 更新VCR配置 * @param vcrConfig vcr配置 * @return 是否发送成功,不代表是否生效 * * * Update VCR configuration * @param vcrConfig * @return Whether successful */ updateVcrConfig(vcrConfig: AICallAgentVcrConfig): boolean; /** * Vision智能体,开始启动自定义截帧,启动后,无法通过语音与智能体通话 */ startVisionCustomCapture(req: AICallVisionCustomCaptureRequest): boolean | undefined; /** * 更新百炼应用中心参数 * @param params 参数 * @return 是否发送成功,不代表是否生效 * * * Update the Bailian App Center parameters * @param params parameters * @return Whether successful */ updateBailianAppParams(params: JSONObject): boolean; enableVoiceprint(enable: boolean): boolean; /** * Vision智能体,结束自定义截帧 */ stopVisionCustomCapture(): boolean | undefined; generateShareAgentCall(shareConfig: AICallAgentShareConfig, userId: string): Promise; private sendMsgToDataChannel; } export default ARTCAICallEngine; export declare const defaultVisionCustomCaptureRequest: { text: string; isSingle: boolean; eachDuration: number; num: number; duration: number; userData: string; }; declare type EquipmentResult = { mobilePhoneCount: number; headPhoneCount: number; watchCount: number; }; /** * Minimal `EventEmitter` interface that is molded against the Node.js * `EventEmitter` interface. */ declare class EventEmitter< EventTypes extends EventEmitter.ValidEventTypes = string | symbol, Context extends any = any > { static prefixed: string | boolean; /** * Return an array listing the events for which the emitter has registered * listeners. */ eventNames(): Array>; /** * Return the listeners registered for a given event. */ listeners>( event: T ): Array>; /** * Return the number of listeners listening to a given event. */ listenerCount(event: EventEmitter.EventNames): number; /** * Calls each of the listeners registered for a given event. */ emit>( event: T, ...args: EventEmitter.EventArgs ): boolean; /** * Add a listener for a given event. */ on>( event: T, fn: EventEmitter.EventListener, context?: Context ): this; addListener>( event: T, fn: EventEmitter.EventListener, context?: Context ): this; /** * Add a one-time listener for a given event. */ once>( event: T, fn: EventEmitter.EventListener, context?: Context ): this; /** * Remove the listeners of a given event. */ removeListener>( event: T, fn?: EventEmitter.EventListener, context?: Context, once?: boolean ): this; off>( event: T, fn?: EventEmitter.EventListener, context?: Context, once?: boolean ): this; /** * Remove all listeners, or those of the specified event. */ removeAllListeners(event?: EventEmitter.EventNames): this; } declare namespace EventEmitter { interface ListenerFn { (...args: Args): void; } interface EventEmitterStatic { new < EventTypes extends ValidEventTypes = string | symbol, Context = any >(): EventEmitter; } /** * `object` should be in either of the following forms: * ``` * interface EventTypes { * 'event-with-parameters': any[] * 'event-with-example-handler': (...args: any[]) => void * } * ``` */ type ValidEventTypes = string | symbol | object; type EventNames = T extends string | symbol ? T : keyof T; type ArgumentMap = { [K in keyof T]: T[K] extends (...args: any[]) => void ? Parameters : T[K] extends any[] ? T[K] : any[]; }; type EventListener< T extends ValidEventTypes, K extends EventNames > = T extends string | symbol ? (...args: any[]) => void : ( ...args: ArgumentMap>[Extract] ) => void; type EventArgs< T extends ValidEventTypes, K extends EventNames > = Parameters>; const EventEmitter: EventEmitterStatic; } declare type FrameMotionResult = { duration: number; }; declare type HeadMotionResult = { nodDetected: boolean; shakeDetected: boolean; }; declare interface ImAttachmentProgress { progress: number; totalSize: number; currentSize: number; } declare interface ImAttachmentReq { id: string; fileType: ImAttachmentType; fileName: string; file?: File | Blob; filePath?: string; extra?: string; onProgress?: (res: ImAttachmentProgress) => void; } declare interface ImAttachmentRes { id: string; fileType: ImAttachmentType; fileSize: number; fileName: string; accessKey: string; fileDuration: number; extra: string; } declare enum ImAttachmentType { IMAGE = 1, AUDIO = 2, VIDEO = 3, OTHER = 4 } declare class ImStreamMessageSender extends EventEmitter { private wasmIns; private sender; constructor(wasmIns: any); setSender(sender: any): void; getMessageId(): string; /** * 发送字节数据 * @param {Uint8Array} byteData 字节数据 * @param {boolean} isLast 是否结束流 * @param {ImAttachmentRes[]} [attachments] 附件列表 */ sendByteData(byteData: Uint8Array, isLast: boolean, attachments?: ImAttachmentRes[]): void; /** * 取消发送 * @param {number} [code] 取消码 * @param {string} [msg] 取消原因 */ cancel(code?: number, msg?: string): void; destroy(): void; } declare interface ImStreamMessageSenderListener { /** * 流消息结束通知 * @param {string} messageId 消息ID * @param {number} endCode 结束原因:0正常处理结束,1主动取消,2与智能体服务连接异常断开,3连接超时断开,4收到新的开始切片,5包请求异常 * @param {number} [subCode] 详情码 * @param {string} [subMsg] 详情信息 */ streammessageend: (messageId: string, endCode: number, subCode?: number, subMsg?: string) => void; } export declare interface JSONObject { [key: string]: JSONPrimitive | JSONPrimitive[] | JSONObject | JSONObject[]; } declare type JSONPrimitive = string | number | boolean | null; export declare type LatencyStat = { AIAgent: { aiasrcost?: number; aillmcost?: number; aittscost?: number; aimutecost?: number; aiagenttotalcost?: number; aiagentextraavatarcost?: number; }; AIPub: { aAIClientQuestionTimestampHEAD?: number; aAIClientQuestionTimestampTAIL?: number; aAIClientAnswerTimestampHEAD?: number; aAIClientAnswerTimestampTAIL?: number; aAIClientScentenceId?: number; aAIClientSourceCost?: number; aAIClientMixerCost?: number; aAIClientEncoderCost?: number; aIClientNetsdkThrCost?: number; aIClientQosThrCost?: number; aIClientPacerCost?: number; aIClientUpRTTHalf?: number; }; AISub: { aAIClientQuestionUidDigestHEAD?: number; aAIClientQuestionUidDigestTAIL?: number; aAILinuxReceiveCost?: number; aAILinuxNeteqCost?: number; aAILinuxRemoteSourceCost?: number; aAILinuxPlayMixerCost?: number; aAILinuxPlayCost?: number; aIClinetSmartDenoise?: number; }; }; declare type LookAwayResult = { count: number; duration: number; }; declare type PeopleCountResult = { count: number; }; export { }