/** * Backend-specific types for the vLLM backend implementation. * Extends / augments the shared @omnius/schemas types. */ import type { AgentMessage } from "../schemas/messages.js"; /** * Backend-specific types for the vLLM backend implementation. * Extends / augments the shared @omnius/schemas types. */ import type { ToolCall } from "../schemas/tools.js"; import type { ProviderProtocol } from "./providerRegistry.js"; export interface VllmBackendConfig { /** Provider endpoint URL. Known providers are detected from this URL. */ baseUrl: string; /** Model name as registered in vLLM */ model: string; /** Optional Bearer token for authenticated deployments */ apiKey?: string; /** Request timeout in milliseconds (default: 60_000) */ timeoutMs?: number; /** Maximum retries on transient network errors (default: 3) */ maxRetries?: number; /** Stable provider id or alias, for example gemini, anthropic, or openai. */ providerId?: string; /** * Required for unknown/custom endpoints. Known providers may omit it, but a * supplied value must agree with the registry. */ protocol?: ProviderProtocol; /** Deterministic migration hint for pre-protocol saved configurations. */ legacyBackendType?: string; /** Omnius client version inserted into provider-specific telemetry headers. */ clientVersion?: string; } /** Kept for backwards-compat with the existing test that imports VllmClient */ export interface VllmConfig { baseUrl: string; apiKey?: string; model: string; maxTokens?: number; temperature?: number; timeout?: number; providerId?: string; protocol?: ProviderProtocol; legacyBackendType?: string; clientVersion?: string; } export interface ChatMessage { role: "system" | "user" | "assistant" | "tool"; content: string | null; /** Tool call id when role === "tool" */ tool_call_id?: string; /** Only present when role === "assistant" and the model requested tool calls */ tool_calls?: WireToolCall[]; name?: string; } export interface WireToolCall { id: string; type: "function"; function: { name: string; /** JSON-serialised arguments */ arguments: string; }; } export interface ToolDefinition { type: "function"; function: { name: string; description?: string; parameters: Record; }; } export interface ResponseFormat { type: "json_schema"; json_schema: { name: string; strict?: boolean; schema: Record; }; } export interface ChatCompletionRequest { model: string; messages: ChatMessage[]; temperature?: number; max_tokens?: number; tools?: ToolDefinition[]; tool_choice?: "auto" | "none" | "required" | { type: "function"; function: { name: string; }; }; response_format?: ResponseFormat; } export interface ChatCompletionChoice { index: number; message: ChatMessage; finish_reason: "stop" | "tool_calls" | "length" | "content_filter" | null; } export interface UsageStats { prompt_tokens: number; completion_tokens: number; total_tokens: number; cache_creation_input_tokens?: number; cache_read_input_tokens?: number; prompt_eval_count?: number; } export interface ChatCompletionResponse { id: string; object: "chat.completion"; created: number; model: string; choices: ChatCompletionChoice[]; usage: UsageStats; } export interface CompletionRequest { messages: AgentMessage[]; tools?: ToolDefinition[]; responseFormat?: ResponseFormat; temperature?: number; maxTokens?: number; /** * Routing hint: prefix hash computed from the conversation prefix. * Used by the router for APC (Automatic Prefix Caching) optimisation. */ prefixHash?: string; /** * Optional stable memory prefix produced by a memory prefill adapter. * Backends without raw KV injection may still benefit from identical * prefixes through their native prompt-prefix cache. */ memoryPrefix?: string; /** * Weighted memory prefill references. These require backend-specific raw * KV/prefill support; OpenAI-compatible HTTP backends generally ignore them. */ weightedMemoryPrefill?: Array<{ itemId: string; chunkId: string; weight: number; }>; } export interface CompletionResult { /** Full text response when no tool calls were made */ content: string | null; /** Tool calls requested by the model */ toolCalls: ToolCall[]; /** Whether the model's output was cut short due to max_tokens */ truncated: boolean; usage: UsageStats; /** Wall-clock milliseconds elapsed for the request */ latencyMs: number; } export interface LLMBackend { /** Returns true when the backend is reachable and the model is loaded */ healthCheck(): Promise; /** Perform a chat completion */ complete(request: CompletionRequest): Promise; /** Report backend memory prefill support, if implemented. */ memoryPrefillCapabilities?(): BackendMemoryPrefillCapability; /** Put the backend into sleep / low-power mode */ sleep(): Promise; /** Wake the backend from sleep mode */ wake(): Promise; } export interface BackendMemoryPrefillCapability { backend: string; supportsRawKvPrefill: boolean; supportsPrefixCache: boolean; supportsWeightedPrefill: boolean; strategy: "raw_kv" | "prefix_cache" | "prompt_text" | "none"; reason?: string; } export interface MetricsSnapshot { /** Running GPU KV-cache utilisation in the range [0, 1] */ gpuCacheUsage: number; /** Number of requests currently waiting in the scheduler queue */ numWaitingRequests: number; /** Number of requests currently being processed */ numRunningRequests: number; /** Total requests fulfilled since the server started */ totalRequestsCompleted: number; /** Raw Prometheus text (unparsed) */ rawPrometheus: string; } export interface RoutingDecision { /** Selected replica base URL */ targetUrl: string; /** The prefix hash used for the routing decision */ prefixHash: string; /** All candidate URLs that were considered */ candidates: string[]; } export interface CompletionResponse { id: string; content: string; toolCalls?: Array<{ id: string; name: string; arguments: string; }>; usage: { promptTokens: number; completionTokens: number; totalTokens: number; }; finishReason: "stop" | "tool_calls" | "length" | "error"; }