import { ZodSchema, ZodInfer } from '../../zod/dist/index.js'; import { Logger } from '@llamaindex/env'; import { Tokenizers } from '@llamaindex/env/tokenizers'; import { JSONSchemaType } from 'ajv'; import { JSONValue, JSONObject } from '../../global/dist/index.js'; import { ModalityType } from '../../schema/dist/index.js'; /** * @internal */ interface LLMChat { chat(params: LLMChatParamsStreaming | LLMChatParamsNonStreaming): Promise | AsyncIterable>>; } /** * Unified language model interface */ interface LLM extends LLMChat { metadata: LLMMetadata; /** * Get a chat response from the LLM */ chat(params: LLMChatParamsStreaming): Promise>; chat(params: LLMChatParamsNonStreaming): Promise>; /** * Get a prompt completion from the LLM */ complete(params: LLMCompletionParamsStreaming): Promise>; complete(params: LLMCompletionParamsNonStreaming): Promise; } type MessageType = "user" | "assistant" | "system" | "memory" | "developer"; type TextChatMessage = { content: string; role: MessageType; options?: undefined | AdditionalMessageOptions; }; type ChatMessage = { content: MessageContent; role: MessageType; options?: undefined | AdditionalMessageOptions; }; interface ChatResponse { message: ChatMessage; /** * Raw response from the LLM * * If LLM response an iterable of chunks, this will be an array of those chunks */ raw: object | null; } type ChatResponseChunk = { raw: object | null; delta: string; options?: undefined | AdditionalMessageOptions; }; interface ExecResponse { newMessages: ChatMessage[]; toolCalls: ToolCall[]; object?: O | undefined; } interface ExecStreamResponse { stream: AsyncIterable>; newMessages(): ChatMessage[]; toolCalls: ToolCall[]; object?: O | undefined; } interface CompletionResponse { text: string; /** * Raw response from the LLM * * It's possible that this is `null` if the LLM response an iterable of chunks */ raw: object | null; } type LLMMetadata = { model: string; temperature: number; topP: number; maxTokens?: number | undefined; contextWindow: number; tokenizer: Tokenizers | undefined; structuredOutput: boolean; }; interface LLMChatParamsBase { messages: ChatMessage[]; additionalChatOptions?: AdditionalChatOptions | undefined; tools?: BaseTool[] | undefined; responseFormat?: Schema | object | undefined; logger?: Logger | undefined; } interface LLMChatParamsStreaming extends LLMChatParamsBase { stream: true; } interface LLMChatParamsNonStreaming extends LLMChatParamsBase { stream?: false; } interface LLMCompletionParamsBase { prompt: MessageContent; responseFormat?: ZodSchema | object; } interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase { stream: true; } interface LLMCompletionParamsNonStreaming extends LLMCompletionParamsBase { stream?: false | null | undefined; } type MessageContentTextDetail = { type: "text"; text: string; }; type MessageContentImageDetail = { type: "image_url"; image_url: { url: string; }; detail?: "high" | "low" | "auto"; }; type MessageContentAudioDetail = { type: "audio"; data: string; mimeType: string; }; type MessageContentVideoDetail = { type: "video"; data: string; mimeType: string; }; type MessageContentImageDataDetail = { type: "image"; data: string; mimeType: string; }; type MessageContentFileDetail = { type: "file"; data: string; mimeType: string; }; type MessageContentDetail = MessageContentTextDetail | MessageContentImageDetail | MessageContentAudioDetail | MessageContentVideoDetail | MessageContentImageDataDetail | MessageContentFileDetail; /** * Extended type for the content of a message that allows for multi-modal messages. */ type MessageContent = string | MessageContentDetail[]; type ToolCall = { name: string; input: JSONObject; id: string; }; type PartialToolCall = { name: string; id: string; input: string; }; type ToolResult = { id: string; result: string; isError: boolean; }; type ToolCallOptions = { toolCall: (ToolCall | PartialToolCall)[]; }; type ToolResultOptions = { toolResult: ToolResult; }; type ToolCallLLMMessageOptions = ToolResultOptions | ToolCallOptions | object; type Known = { [key: string]: Known; } | [Known, ...Known[]] | Known[] | number | string | boolean | null; type ToolMetadata = Record> = { description: string; name: string; /** * OpenAI uses JSON Schema to describe the parameters that a tool can take. * @link https://json-schema.org/understanding-json-schema */ parameters?: Parameters; }; /** * Simple Tool interface. Likely to change. */ interface BaseTool { /** * This could be undefined if the implementation is not provided, * which might be the case when communicating with a llm. * * @return {JSONValue | Promise} The output of the tool. */ call?: (input: Input) => JSONValue | Promise; metadata: Input extends Known ? ToolMetadata> : ToolMetadata; } type BaseToolWithCall = Omit, "call"> & { call: NonNullable, "call">["call"]>; }; type ToolOutput = { tool: BaseTool | undefined; input: JSONObject; output: JSONValue; isError: boolean; }; interface AudioConfig { stream?: MediaStream; onTrack?: (track: MediaStream | null) => void; } interface LiveConnectConfig { tools?: BaseTool[]; responseModality?: ModalityType[]; systemInstruction?: string; audioConfig?: AudioConfig; } declare abstract class BaseLLM implements LLM { abstract metadata: LLMMetadata; complete(params: LLMCompletionParamsStreaming): Promise>; complete(params: LLMCompletionParamsNonStreaming): Promise; abstract chat(params: LLMChatParamsStreaming): Promise>>; abstract chat(params: LLMChatParamsNonStreaming): Promise>; exec(params: LLMChatParamsStreaming): Promise>>; exec(params: LLMChatParamsNonStreaming): Promise>>; streamExec(params: LLMChatParamsStreaming): Promise>>; } declare abstract class ToolCallLLM extends BaseLLM { abstract supportToolCall: boolean; } type OpenEvent = { type: "open"; }; type AudioEvent = MessageContentAudioDetail; type TextEvent = MessageContentTextDetail; type ErrorEvent = { type: "error"; error: unknown; }; type CloseEvent = { type: "close"; }; type SetupCompleteEvent = { type: "setupComplete"; }; type InterruptedEvent = { type: "interrupted"; }; type GenerationCompleteEvent = { type: "generationComplete"; }; type TurnCompleteEvent = { type: "turnComplete"; }; type LiveEvent = OpenEvent | AudioEvent | TextEvent | ErrorEvent | CloseEvent | SetupCompleteEvent | InterruptedEvent | GenerationCompleteEvent | TurnCompleteEvent; declare const liveEvents: { open: { include: (e: LiveEvent) => e is OpenEvent; }; audio: { include: (e: LiveEvent) => e is AudioEvent; }; text: { include: (e: LiveEvent) => e is TextEvent; }; error: { include: (e: LiveEvent) => e is ErrorEvent; }; close: { include: (e: LiveEvent) => e is CloseEvent; }; setupComplete: { include: (e: LiveEvent) => e is SetupCompleteEvent; }; interrupted: { include: (e: LiveEvent) => e is InterruptedEvent; }; generationComplete: { include: (e: LiveEvent) => e is GenerationCompleteEvent; }; turnComplete: { include: (e: LiveEvent) => e is TurnCompleteEvent; }; }; interface MessageSender { sendTextMessage(message: string, role?: string): void; sendAudioMessage?(content: MessageContentAudioDetail, role?: string): void; sendImageMessage?(content: MessageContentImageDataDetail, role?: string): void; sendVideoMessage?(content: MessageContentVideoDetail, role?: string): void; } declare enum LiveLLMCapability { EPHEMERAL_KEY = "ephemeral_key", AUDIO_CONFIG = "audio_config" } declare abstract class LiveLLMSession { protected eventQueue: LiveEvent[]; protected eventResolvers: ((value: LiveEvent) => void)[]; closed: boolean; abstract get messageSender(): MessageSender; private isTextMessage; private isAudioMessage; private isImageMessage; private isVideoMessage; sendMessage(message: ChatMessage): void; private processMessage; streamEvents(): AsyncIterable; abstract disconnect(): Promise; protected nextEvent(): Promise; pushEventToQueue(event: LiveEvent): void; } declare abstract class LiveLLM { /** * Set of capabilities supported by this implementation. * Override in subclasses as needed. */ capabilities: Set; abstract connect(config?: LiveConnectConfig): Promise; abstract getEphemeralKey(): Promise; hasCapability(capability: LiveLLMCapability): boolean; } declare function addContentPart(message: ChatMessage, part: MessageContentTextDetail | MessageContentImageDataDetail): void; export { BaseLLM, LiveLLM, LiveLLMCapability, LiveLLMSession, ToolCallLLM, addContentPart, liveEvents }; export type { AudioConfig, BaseTool, BaseToolWithCall, ChatMessage, ChatResponse, ChatResponseChunk, CompletionResponse, LLM, LLMChat, LLMChatParamsBase, LLMChatParamsNonStreaming, LLMChatParamsStreaming, LLMCompletionParamsBase, LLMCompletionParamsNonStreaming, LLMCompletionParamsStreaming, LLMMetadata, LiveConnectConfig, LiveEvent, MessageContent, MessageContentAudioDetail, MessageContentDetail, MessageContentFileDetail, MessageContentImageDataDetail, MessageContentImageDetail, MessageContentTextDetail, MessageContentVideoDetail, MessageSender, MessageType, PartialToolCall, TextChatMessage, ToolCall, ToolCallLLMMessageOptions, ToolCallOptions, ToolMetadata, ToolOutput, ToolResult, ToolResultOptions };