/** * LLM - Configuration surface and an OpenAI-compatible base client. * * Python parity with praisonaiagents/llm/llm.py `LLM.__init__`. Every * constructor parameter of the Python class is accepted here and honoured: * * - Request-shaped options reach the Chat Completions body built by * {@link BaseLLM.buildRequestParams}. * - Transport-shaped options reach the client built by * {@link BaseLLM.buildClientOptions}. * - `maxIter` bounds the tool-calling loop in {@link BaseLLM.generateWithTools}. * - `events` are listeners invoked on request start/end, tool calls, errors * and retries (Python: litellm callbacks + display callbacks). * - `failoverManager` rotates credentials on retryable failures. * - `promptCaching`, `webFetch` and `claudeMemory` emit the Anthropic-specific * request shapes Python emits, and are rejected on models that cannot * carry them rather than silently ignored. * - `auth` resolves credentials from an object, a function or a registered * named provider. */ import OpenAI from 'openai'; import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions'; import { FunctionTool, type ToolConfig, type ToolContext } from '../tools/decorator'; /** * Event names. `llm_start`, `tool_call`, `error` and `retry` are the display * callback names Python fires from llm.py; `llm_end` and `tool_result` are * their natural counterparts. */ export type LLMEventName = 'llm_start' | 'llm_end' | 'tool_call' | 'tool_result' | 'error' | 'retry'; export interface LLMEvent { type: LLMEventName; model: string; /** `Date.now()` when the event fired. */ timestamp: number; /** Request body (`llm_start`, `llm_end`, `error`, `retry`). */ request?: Record; /** Raw provider response (`llm_end`). */ response?: unknown; /** Wall-clock duration of the request in ms (`llm_end`, `error`). */ durationMs?: number; /** The failure (`error`, `retry`). */ error?: unknown; /** Tool call details (`tool_call`, `tool_result`). */ toolName?: string; toolCallId?: string; arguments?: unknown; result?: unknown; /** Retry bookkeeping (`retry`), mirroring Python's `execute_sync_callback('retry', ...)`. */ attempt?: number; maxAttempts?: number; retryInSeconds?: number; /** Name of the failover profile now in use (`retry`). */ profile?: string; } /** * Object listener. Either Python display-callback method names (`llm_start`, * `tool_call`, ...) or litellm `CustomLogger` hook names * (`log_pre_api_call`, `log_success_event`, `log_failure_event`) - both are * what Python's `events` list accepts. */ export interface LLMEventHooks { llm_start?(event: LLMEvent): void | Promise; llm_end?(event: LLMEvent): void | Promise; tool_call?(event: LLMEvent): void | Promise; tool_result?(event: LLMEvent): void | Promise; error?(event: LLMEvent): void | Promise; retry?(event: LLMEvent): void | Promise; log_pre_api_call?(model: string, messages: unknown, kwargs: Record): void | Promise; log_success_event?(kwargs: Record, response: unknown, startTime: number, endTime: number): void | Promise; log_failure_event?(kwargs: Record, error: unknown, startTime: number, endTime: number): void | Promise; } export type LLMEventListener = ((event: LLMEvent) => void | Promise) | LLMEventHooks; /** A credential profile, mirroring Python's `AuthProfile` fields that affect a request. */ export interface LLMFailoverProfile { name?: string; provider?: string; apiKey?: string; baseURL?: string; /** Model override while this profile is active. */ model?: string; headers?: Record; } /** * Minimal failover manager contract. Both Python's * `FailoverManagerProtocol` (`getNextProfile` / `markFailure` / `markSuccess`) * and the gateway `FailoverManager` (`getNextProvider` / `markFailed` / * `markHealthy`) satisfy it. A profile returned as a bare string is treated * as a name only; return an object with `apiKey` / `baseURL` / `model` to * actually switch credentials. */ export interface LLMFailoverManager { getNextProfile?(): LLMFailoverProfile | string | null | undefined; getNextProvider?(): string | null | undefined; markFailure?(profile: LLMFailoverProfile, error: string, isRateLimit?: boolean): void; markFailed?(provider: string): void; markSuccess?(profile: LLMFailoverProfile): void; markHealthy?(provider: string, latency?: number): void; } /** Python parity: `LLM.classify_error_kind`. */ export type LLMErrorKind = 'auth_permanent' | 'auth' | 'billing' | 'rate_limit' | 'model_not_found' | 'context_overflow' | 'format_error' | 'overloaded' | 'connection_error' | 'unknown'; export interface LLMFailoverDecision { action: 'retry' | 'rotate_profile' | 'surface_error'; reason: LLMErrorKind; backoffMs: number; isRetryable: boolean; } /** Map a provider/network error onto Python's error kinds. */ export declare function classifyLLMError(error: unknown): LLMErrorKind; export { registerAuthProvider, listAuthProviders, resetAuthProviders, resolveAuth, } from './auth'; export type { LLMAuth, LLMAuthResolver, LLMAuthSource } from './auth'; import type { LLMAuthSource } from './auth'; /** Anthropic-style model detection (Python `_is_anthropic_model`). */ export declare function isAnthropicModel(model: string | undefined): boolean; /** Python parity: `LLM.max_iter` default (`max_iter or 20`). */ export declare const DEFAULT_MAX_ITER = 20; /** Python parity: `extra_settings['max_retries']` default. */ export declare const DEFAULT_MAX_RETRIES = 3; /** Python parity: `LLM.STEP_LIMIT_WRAP_UP_PROMPT`. */ export declare const STEP_LIMIT_WRAP_UP_PROMPT: string; /** Python parity: `LLM.STEP_LIMIT_FALLBACK_MESSAGE`. */ export declare const STEP_LIMIT_FALLBACK_MESSAGE = "Reached the step limit before finishing this task."; /** Python parity: `ClaudeMemoryTool.BETA_HEADER`. */ export declare const CLAUDE_MEMORY_BETA_HEADER = "context-management-2025-06-27"; /** Python parity: `ClaudeMemoryTool.TOOL_DEFINITION`. */ export declare const CLAUDE_MEMORY_TOOL_DEFINITION: { readonly type: "memory_20250818"; readonly name: "memory"; }; /** * A caller-supplied Claude memory backend (Python: a `ClaudeMemoryTool` * instance). `getToolDefinition` / `getBetaHeader` default to the Anthropic * memory tool; `execute` handles the model's `memory` tool calls. */ export interface ClaudeMemoryBackend { getToolDefinition?(): Record; getBetaHeader?(): string; execute?(input: Record): unknown | Promise; } export interface LLMConfig { model: string; temperature?: number; maxTokens?: number; apiKey?: string; baseURL?: string; /** Python parity: timeout (Optional[int], default None). Seconds; applied to the client and every request. */ timeout?: number; /** Python parity: top_p (Optional[float], default None). Sent as `top_p`. */ topP?: number; /** Python parity: n (Optional[int], default None). Number of completions; sent as `n`. */ n?: number; /** Python parity: presence_penalty (Optional[float], default None). Sent as `presence_penalty`. */ presencePenalty?: number; /** Python parity: frequency_penalty (Optional[float], default None). Sent as `frequency_penalty`. */ frequencyPenalty?: number; /** Python parity: logit_bias (Optional[Dict[int, float]], default None). Sent as `logit_bias`. */ logitBias?: Record; /** Python parity: response_format (Optional[Dict[str, Any]], default None). Sent as `response_format`. */ responseFormat?: Record; /** Python parity: seed (Optional[int], default None). Sent as `seed`. */ seed?: number; /** Python parity: logprobs (Optional[bool], default None). Sent as `logprobs`. */ logprobs?: boolean; /** Python parity: top_logprobs (Optional[int], default None). Sent as `top_logprobs`. */ topLogprobs?: number; /** Python parity: api_version (Optional[str], default None). Sent as the `api-version` query parameter (Azure convention). */ apiVersion?: string; /** Python parity: stop_phrases (Optional[Union[str, List[str]]], default None). Sent as `stop`. */ stopPhrases?: string | string[]; /** * Python parity: events (List[Any], default []). Listeners invoked on * `llm_start`, `llm_end`, `tool_call`, `tool_result`, `error` and `retry`. * Functions receive an {@link LLMEvent}; objects may implement those names * or litellm's `log_pre_api_call` / `log_success_event` / `log_failure_event`. */ events?: LLMEventListener[]; /** Python parity: web_search (Optional[Union[bool, Dict[str, Any]]], default None). Sent as `web_search_options` (`true` -> medium context size). */ webSearch?: boolean | Record; /** * Python parity: web_fetch (Optional[Union[bool, Dict[str, Any]]], default None). * Adds Anthropic's `web_fetch_20250910` tool to every request on an * Anthropic-style model (`true` -> `max_uses: 5`; a dict passes * `max_uses` / `allowed_domains` / `blocked_domains` / `citations` / * `max_content_tokens`). Throws on other models. */ webFetch?: boolean | Record; /** * Python parity: prompt_caching (Optional[bool], default None). On * Anthropic-style models, emits `cache_control: { type: 'ephemeral' }` * breakpoints on the system prompt and the end of the stable history * prefix (all but the two most recent messages), exactly as Python's * `_build_messages` does. OpenAI-compatible endpoints cache prefixes * automatically, so nothing is emitted for them. */ promptCaching?: boolean; /** * Python parity: claude_memory (Optional[Union[bool, Any]], default None). * On Anthropic-style models, adds the `memory_20250818` tool and the * `anthropic-beta: context-management-2025-06-27` header; `memory` tool * calls in the tool loop are dispatched to a supplied * {@link ClaudeMemoryBackend}. Throws on other models. */ claudeMemory?: boolean | ClaudeMemoryBackend; /** * Python parity: failover_manager (Optional[FailoverManagerProtocol], default None). * On a retryable failure (rate limit, 5xx, network, retryable auth) the * manager is asked for the next profile and the request is retried with * its credentials, up to `DEFAULT_MAX_RETRIES` times. */ failoverManager?: LLMFailoverManager; /** * Python parity: auth (Optional[str], default None). Credentials resolved * through {@link resolveAuth}: `{ apiKey, baseURL, headers }`, a function * returning that, or a name registered with {@link registerAuthProvider}. */ auth?: LLMAuthSource; /** Python parity: max_iter (Optional[int], default None -> 20). Tool-calling loop budget for {@link BaseLLM.generateWithTools}. */ maxIter?: number; } export interface LLMResponse { text: string; usage?: { promptTokens: number; completionTokens: number; totalTokens: number; }; metadata?: Record; } export interface LLM { config: LLMConfig; generate(prompt: string): Promise; generateStream(prompt: string): AsyncGenerator; } /** A tool the loop can execute: a {@link FunctionTool} or its config. */ export type LLMTool = FunctionTool | ToolConfig; /** Per-call options for {@link BaseLLM.generate} / {@link BaseLLM.generateStream}. */ export interface LLMGenerateOptions { /** Optional system prompt prepended to the conversation. */ systemPrompt?: string; /** Abort signal for cancellation - aborts the underlying HTTP request. */ signal?: AbortSignal; /** Tools to offer; when present `generate()` runs the tool-calling loop. */ tools?: LLMTool[]; /** Overrides `config.maxIter` for this call. */ maxIter?: number; } /** Per-call options for {@link BaseLLM.generateWithTools}. */ export interface LLMToolLoopOptions { signal?: AbortSignal; /** Overrides `config.maxIter` for this call (Python: `max_iterations` kwarg). */ maxIter?: number; /** Context handed to every tool execution. */ toolContext?: ToolContext; } export interface LLMToolCallRecord { id: string; name: string; arguments: unknown; result: unknown; iteration: number; } export interface LLMToolLoopResult extends LLMResponse { /** Full conversation including assistant tool calls and tool results. */ messages: ChatCompletionMessageParam[]; /** Number of model calls that returned tool calls. */ iterations: number; toolCalls: LLMToolCallRecord[]; /** Python parity: `_last_stop_reason`. */ stopReason: 'completed' | 'max_steps'; } /** Request options handed to the OpenAI SDK alongside the body. */ export interface LLMRequestOptions { signal?: AbortSignal; /** Milliseconds. */ timeout?: number; headers?: Record; } /** * OpenAI-compatible LLM client that honours every {@link LLMConfig} field. * Works against any OpenAI-compatible `baseURL` (OpenAI, Azure with * `apiVersion`, Ollama, vLLM, OpenRouter, LiteLLM proxies, ...). * * @example * const llm = new BaseLLM({ model: 'gpt-4o-mini', topP: 0.9, seed: 7, stopPhrases: ['END'] }); * const { text } = await llm.generate('Say hi'); */ export declare class BaseLLM implements LLM { config: LLMConfig; private client; private profileClients; private listeners; private resolvedAuth; private initialProfile; constructor(config: LLMConfig); /** * `webFetch` and `claudeMemory` only exist as Anthropic request shapes. On * any other model Python silently drops them; here that would be an * accepted-and-ignored setting, so they are rejected up front. */ private validateAnthropicOnlyOptions; /** Add a listener (EventEmitter-style). Returns `this`. */ on(listener: LLMEventListener): this; /** Remove a previously added listener. */ off(listener: LLMEventListener): this; /** Listeners currently registered (the `events` config plus `on()` additions). */ get eventListeners(): readonly LLMEventListener[]; /** * Fire an event. Listener failures are logged, never propagated: a * misbehaving observer must not fail the request (litellm does the same). */ protected emit(event: Omit & { model?: string; }): void; private warnListener; /** * Client construction options derived from the config: credentials, * `baseURL`, `timeout` (seconds -> milliseconds), the Azure-style * `api-version` query parameter, resolved `auth` credentials and, when * given, a failover profile's overrides. */ buildClientOptions(profile?: LLMFailoverProfile | null): Record; /** Per-request options: abort signal, configured timeout (ms) and any beta headers. */ buildRequestOptions(signal?: AbortSignal): LLMRequestOptions; /** Resolve `config.auth` once (Python caches subscription credentials too). */ private ensureAuth; /** The lazily constructed OpenAI-compatible client (default credentials). */ getClient(): Promise; /** A client for a failover profile, cached per profile. */ private getClientFor; /** Python parity: `_explicit_cache_breakpoints()`. */ private explicitCacheBreakpoints; /** Python parity: `_mark_message_cache_control` on a shallow copy. Returns the copy, or null if unmarkable. */ private static markCacheControl; /** Python parity: `_count_cache_markers`. */ private static countCacheMarkers; /** * Anthropic cache breakpoints, placed where Python's `_build_messages` * places them: every system message becomes a cached content block, and * one breakpoint marks the end of the stable history prefix (all but the * two most recent messages), within the provider's 4-marker budget. * Caller-owned messages are never mutated. */ applyPromptCaching(messages: ChatCompletionMessageParam[]): ChatCompletionMessageParam[]; /** Python parity: the `web_fetch` tool dict built in `_build_completion_params`. */ webFetchToolDefinition(): Record | null; private claudeMemoryBackend; /** Python parity: `ClaudeMemoryTool.get_tool_definition()`. */ claudeMemoryToolDefinition(): Record | null; /** Python parity: `ClaudeMemoryTool.get_beta_header()`. */ claudeMemoryBetaHeader(): string | null; /** * The Chat Completions request body for `messages`, carrying every * request-shaped config field under its wire name. Only fields that were * set are included so provider defaults stay in force. `tools` are the * function tools to offer; Anthropic server tools (`web_fetch`, `memory`) * are appended when configured, as Python does. */ buildRequestParams(messages: ChatCompletionMessageParam[], tools?: unknown[]): Record; private buildMessages; /** Overridable for tests; mirrors Python's `time.sleep(retry_delay)`. */ protected sleep(ms: number): Promise; /** Python parity: `resolve_failover_decision`. `attempt` is 1-based. */ resolveFailoverDecision(error: unknown, attempt: number, maxRetries?: number): LLMFailoverDecision; private managerNextProfile; private managerMarkFailure; private managerMarkSuccess; /** * Python parity: `__init__` takes the first profile from the manager so the * initial request already runs on managed credentials. */ private startingProfile; /** * One Chat Completions call with events and, when a `failoverManager` is * configured, Python's `_call_with_retry` semantics: classify the error, * mark the active profile failed, rotate to the next profile and retry * (up to `DEFAULT_MAX_RETRIES`), surfacing the error when no alternate * profile exists or the error is not retryable. Without a manager the * request is made once (the OpenAI SDK's own transport retries still apply). */ protected completion(body: Record, signal?: AbortSignal): Promise; private toResponse; generate(prompt: string, options?: LLMGenerateOptions): Promise; generateStream(prompt: string, options?: LLMGenerateOptions): AsyncGenerator; /** Python parity: the tool-message content rules in the tool loop. */ private static toolResultContent; private static parseToolArguments; /** * Python parity: `_finalise_on_limit`. One extra call with tools disabled * and the wrap-up nudge; falls back to the last text, then to the fixed * fallback message, if that call fails. */ private finaliseOnLimit; /** * Python parity: the sequential tool-calling loop in `get_response`. * Offers `tools` to the model, executes every tool call it returns, * appends the results and repeats until the model answers without tool * calls or `maxIter` (config `maxIter`, default 20) model calls have * returned tool calls - at which point one final tools-disabled call * summarises what was done (`stopReason: 'max_steps'`). */ generateWithTools(messages: ChatCompletionMessageParam[], tools: LLMTool[], options?: LLMToolLoopOptions): Promise; } export { resolveBackend, resolveBackendSync, isAISDKAvailable, resetAISDKAvailabilityCache, getPreferredBackend, getDefaultModel, parseModelString, type BackendResolutionResult, type BackendSource, type ResolveBackendOptions, } from './backend-resolver'; export { embed, embedMany, createEmbeddingProvider, getDefaultEmbeddingModel, parseEmbeddingModel, cosineSimilarity, euclideanDistance, type EmbeddingOptions, type EmbeddingResult, type EmbeddingBatchResult, } from './embeddings';