import OpenAI from 'openai'; import { TextProvider, PluginLogger, LLMParams, LLMResponse, StreamChunk } from '@quilltap/plugin-types'; /** * OpenAI-Compatible Provider Base Class * * A reusable base class for building LLM providers that use OpenAI-compatible APIs. * This includes services like: * - Local LLM servers (LM Studio, vLLM, Text Generation Web UI, Ollama with OpenAI compat) * - Cloud services with OpenAI-compatible APIs (Gab AI, Together AI, Fireworks, etc.) * * External plugins can extend this class to create custom providers with minimal code: * * @example * ```typescript * import { OpenAICompatibleProvider } from '@quilltap/plugin-utils'; * * export class MyCustomProvider extends OpenAICompatibleProvider { * constructor() { * super({ * baseUrl: 'https://api.my-service.com/v1', * providerName: 'MyService', * requireApiKey: true, * attachmentErrorMessage: 'MyService does not support file attachments', * }); * } * } * ``` * * @packageDocumentation */ /** * Configuration options for OpenAI-compatible providers. * * Use this interface when extending OpenAICompatibleProvider to customize * the provider's behavior for your specific service. */ interface OpenAICompatibleProviderConfig { /** * Base URL for the API endpoint. * Should include the version path (e.g., 'https://api.example.com/v1') */ baseUrl: string; /** * Provider name used for logging context. * This appears in log messages to identify which provider generated them. * @default 'OpenAICompatible' */ providerName?: string; /** * Whether an API key is required for this provider. * If true, requests will fail with an error when no API key is provided. * If false, requests will use 'not-needed' as the API key (for local servers). * @default false */ requireApiKey?: boolean; /** * Custom error message shown when file attachments are attempted. * @default 'OpenAI-compatible provider file attachment support varies by implementation (not yet implemented)' */ attachmentErrorMessage?: string; /** * Default wall-clock budget per request, in milliseconds. * * The OpenAI SDK's own default is 600 000 ms, which means a service that * accepts a connection and then goes quiet holds the caller for ten minutes * before the first retry. That is far longer than any Quilltap caller wants * to wait, so the base class lowers it. * * Individual requests can tighten this further via `LLMParams.requestTimeoutMs`. * * @default 300000 (5 minutes) */ requestTimeoutMs?: number; /** * How many times the SDK may retry a failed or timed-out request. * * Applies only to requests that did *not* specify their own * `LLMParams.requestTimeoutMs` — a caller-supplied budget is a hard ceiling * and never retries. * * @default 2 (the OpenAI SDK default) */ maxRetries?: number; } /** * Base provider class for OpenAI-compatible APIs. * * This class implements the full LLMProvider interface using the OpenAI SDK, * allowing subclasses to create custom providers with just configuration. * * Features: * - Streaming and non-streaming chat completions * - API key validation * - Model listing * - Configurable API key requirements * - Dynamic logging with provider name context * * @remarks * File attachments and image generation are not supported by default, * as support varies across OpenAI-compatible implementations. */ declare class OpenAICompatibleProvider implements TextProvider { /** File attachments are not supported by default */ readonly supportsFileAttachments = false; /** No MIME types are supported for attachments */ readonly supportedMimeTypes: string[]; /** Web search is not supported */ readonly supportsWebSearch = false; /** Base URL for the API endpoint */ protected readonly baseUrl: string; /** Provider name for logging */ protected readonly providerName: string; /** Whether API key is required */ protected readonly requireApiKey: boolean; /** Error message for attachment failures */ protected readonly attachmentErrorMessage: string; /** Default per-request wall-clock budget in milliseconds */ protected readonly requestTimeoutMs: number; /** Retry count for requests that carry no caller-supplied budget */ protected readonly maxRetries: number; /** Logger instance */ protected readonly logger: PluginLogger; /** * Whether the endpoint tolerates more than one leading `system` message. * * True for every hosted service — and so the default, which keeps the * outbound bytes of OpenRouter, Z.AI and every other subclass exactly as they * were. A subclass that points at a local runtime applying the *model's* chat * template sets this false: several template families reject a system message * anywhere but index 0, and Quilltap's context builder deliberately emits up * to three of them so its cache breakpoints survive. See Bug 82 and * {@link collapseLeadingSystemMessages}. */ protected readonly acceptsRepeatedSystemMessages: boolean; /** * Apply {@link acceptsRepeatedSystemMessages} to a mapped wire-message array. * Called by both body builds so the streaming and non-streaming shapes cannot * drift apart. */ protected applySystemMessagePolicy(messages: T[]): T[]; /** * Creates a new OpenAI-compatible provider instance. * * @param config - Configuration object or base URL string (for backward compatibility) */ constructor(config: string | OpenAICompatibleProviderConfig); /** * Collects attachment failures for messages with attachments. * Since attachments are not supported, all attachments are marked as failed. * * @param params - LLM parameters containing messages * @returns Object with empty sent array and failed attachments */ protected collectAttachmentFailures(params: LLMParams): { sent: string[]; failed: { id: string; error: string; }[]; }; /** * Validates that an API key is provided when required. * @throws Error if API key is required but not provided */ protected validateApiKeyRequirement(apiKey: string): void; /** * Gets the effective API key to use for requests. * Returns 'not-needed' for providers that don't require keys. */ protected getEffectiveApiKey(apiKey: string): string; /** * Creates an OpenAI client configured with the provider's base URL, * API key, and Quilltap User-Agent header. * * @param apiKey - API key for authentication * @returns Configured OpenAI client instance */ protected createClient(apiKey: string): OpenAI; /** * Builds the per-request options for a single call. * * When the caller supplied `params.requestTimeoutMs` it is a hard ceiling: * the request uses that budget and does not retry, so the total time spent is * what the caller asked for rather than a multiple of it. Otherwise the * client-level defaults apply. * * Subclasses that build their own request bodies should pass the result as * the second argument to `client.chat.completions.create(...)`. * * @param params - LLM parameters for the request * @returns Request options for the OpenAI SDK, or undefined to use defaults */ protected buildRequestOptions(params: LLMParams): { timeout: number; maxRetries: number; } | undefined; /** * Profile-parameter keys this provider forwards from * `LLMParams.profileParameters` onto the request body. * * **Empty by default**, so a subclass that declares nothing sends exactly the * bytes it sent before this mechanism existed. Subclasses opt in by * overriding with their own allow-list; `model`, `messages`, `stream`, * `stream_options` and `tools` must never appear in one. */ protected readonly profileParamAllowlist: readonly string[]; /** * Per-key hook for allow-listed profile parameters. The default forwards the * stored value verbatim. * * Override to reshape a value (the editor stores flat strings where some APIs * want objects), to gate a key on the model, or — by writing into `body` and * returning `undefined` — to send it under a different key entirely. */ protected normalizeProfileParam(_key: string, value: unknown, _params: LLMParams, _body: Record): unknown | undefined; /** * Apply {@link profileParamAllowlist} to a request body under construction. * Called by both body builds so the streaming and non-streaming shapes cannot * drift apart. */ protected applyProfileParams(body: Record, params: LLMParams): void; /** * Normalize an OpenAI-shaped `tool_calls` array into Quilltap's `ToolCall` * shape, tolerating servers that hand back already-parsed argument objects * where the OpenAI wire format specifies a JSON string. */ protected normalizeToolCalls(toolCalls: readonly unknown[] | undefined | null): { id: string; type: 'function'; function: { name: string; arguments: string; }; }[]; /** * Build the Chat Completions request body — the ONE build both * {@link sendMessage} and {@link streamMessage} call, so the streaming and * non-streaming shapes cannot drift apart. Maps Quilltap messages to the * OpenAI wire format (tool results and assistant tool calls included), * applies {@link applySystemMessagePolicy}, assembles the body literal, and * finishes with {@link applyProfileParams}. * * Subclasses that keep the base class's send/stream loops can override this * to reshape the body in one place. * * @param params - LLM parameters for the request * @param stream - Whether the body is for a streaming request; adds * `stream: true` and `stream_options: { include_usage: true }` */ protected buildRequestBody(params: LLMParams, stream: boolean): Record; /** * Normalizes a Chat Completions `usage` block into Quilltap's shape. * * OpenAI-compatible endpoints fold prompt-cache hits into `prompt_tokens` * (reported separately as `prompt_tokens_details.cached_tokens`). Quilltap's * convention is that `promptTokens` / `totalTokens` EXCLUDE cache reads, so * cached input never counts against token budgets; `cacheUsage` still * carries the cache reads for display. Endpoints that report no * `cached_tokens` are unaffected. */ protected normalizeUsage(usage: OpenAI.Completions.CompletionUsage | null | undefined): Pick; /** * Sends a message and returns the complete response. * * @param params - LLM parameters including messages, model, and settings * @param apiKey - API key for authentication * @returns Complete LLM response with content and usage statistics */ sendMessage(params: LLMParams, apiKey: string): Promise; /** * Sends a message and streams the response. * * @param params - LLM parameters including messages, model, and settings * @param apiKey - API key for authentication * @yields Stream chunks with content and final usage statistics */ streamMessage(params: LLMParams, apiKey: string): AsyncGenerator; /** * Validates an API key by attempting to list models. * * @param apiKey - API key to validate * @returns true if the API key is valid, false otherwise */ validateApiKey(apiKey: string): Promise; /** * Fetches available models from the API. * * @param apiKey - API key for authentication * @returns Sorted array of model IDs, or empty array on failure */ getAvailableModels(apiKey: string): Promise; } /** * Profile-parameter passthrough * * A connection profile's `parameters` blob is free-form JSON: the host stores * whatever the profile editor (or a hand-edited profile) puts there and hands * it back to the provider as `LLMParams.profileParameters` at call time. What * reaches the wire is entirely the provider's business — and a provider that * never reads the bag drops every setting in silence, which is exactly the * defect Bug 71 records. * * This is the one mechanism every provider uses to forward those keys. * * **An exported function rather than a base-class method.** Only one plugin in * the repository extends `OpenAICompatibleProvider` (DeepSeek); Z.AI and * OpenRouter implement `TextProvider` directly, and Ollama speaks its own * REST shape. A `protected` method would have reached one of them, so the * mechanism is a free function usable by inheritance *and* composition, and * the base class simply calls it. * * @packageDocumentation */ /** * Per-key hook applied to an allow-listed profile parameter on its way into * the request body. * * Return the value to send, or `undefined` to omit the key entirely. * * The `body` is passed so a normalizer can place a value somewhere other than * its own top-level key — the OpenAI-compatible plugin folds `reasoning_effort` * into `chat_template_kwargs` this way, then returns `undefined` so no flat * `reasoning_effort` key is sent. Normalizers that only reshape a value can * ignore the argument. */ type ProfileParamNormalizer = (key: string, value: unknown, params: LLMParams, body: Record) => unknown | undefined; /** * Copy allow-listed keys from `params.profileParameters` onto a request body. * * Rules, all of which predate this helper (they were hand-rolled identically in * the DeepSeek and Z.AI providers before it existed): * * - **Allow-list, never spread.** `model`, `messages`, `stream`, * `stream_options` and `tools` must be unreachable from a profile — a * misconfigured bag cannot be allowed to retarget the request. * - **`undefined` and `null` skip.** So does the empty string, which is what * the schema-driven profile editor stores for "omit this and use the model * default". * - **The allow-list is ordered.** Keys are applied in the order given, so a * normalizer that merges into an earlier key (`chat_template_kwargs`) sees * whatever the profile set for it. * * @param body - Request body under construction; mutated in place * @param params - The call's LLM parameters, carrying `profileParameters` * @param allowlist - Keys this provider is willing to forward * @param normalize - Optional per-key hook (see {@link ProfileParamNormalizer}) */ declare function applyProfileParameters(body: Record, params: LLMParams, allowlist: readonly string[], normalize?: ProfileParamNormalizer): void; /** * Leading-system-message collapse. * * Quilltap's context builder deliberately emits the head of a turn as up to * three consecutive `system` messages — the cacheable persona prefix, the * static identity reinforcement, and the compressed-history summary — so that a * cache breakpoint on the first is not invalidated by churn in the others. * Hosted providers accept that happily. * * A local runtime does not answer for itself: it applies the *model's own* chat * template, and several families (Qwen most notably, plus Llama- and * Gemma-derived templates) `raise_exception` on any system message after index * 0. The whole request is then rejected before a token is generated, so the * opening greeting — one system message — works and every turn after it dies. * * The repair belongs here, at request-build time, rather than in the context * assembly: the blocks stay separate for every provider that wants them * separate, and the providers that cannot take them fold the run on the way * out. See Bug 82. * * @packageDocumentation */ /** The shape this helper needs; wire messages carry more, and keep it. */ interface RoleAndContent { role: string; content?: string | null; } /** * Fold a run of consecutive leading `system` messages into a single one, * joining their contents with a blank line. * * Only the *leading* run is touched — a system message that appears later is * left where it is, because moving or merging it would change what the model is * told and when. Arrays with fewer than two leading system messages are * returned unchanged (the same array reference), so a provider that opts in * still sends byte-identical requests everywhere the problem does not arise. * * @param messages - Wire-format messages, already mapped for the endpoint * @returns The messages with their leading system run collapsed */ declare function collapseLeadingSystemMessages(messages: T[]): T[]; /** * Per-request wall-clock budgets. * * Every provider SDK ships a generous default request timeout — the OpenAI and * Anthropic SDKs both use ten minutes — on the assumption that a slow answer is * better than no answer. That assumption is wrong for work Quilltap awaits * inline on a user-visible turn: a service that accepts the connection and then * goes silent holds the turn for the full ten minutes before the SDK's first * retry, with nothing logged, because request logs are only written when a call * finishes. * * `LLMParams.requestTimeoutMs` is the caller's answer to "how long is this * worth waiting for". These helpers are the single place that decides what that * number means, so an SDK-backed provider and a `fetch`-based one apply it the * same way. * * @packageDocumentation */ /** * Fallback budget for a request whose caller expressed no preference. * * Half the SDK default: long enough for a slow non-streaming generation, short * enough that a silently-stalled endpoint fails in minutes rather than tens of * minutes. */ declare const DEFAULT_REQUEST_TIMEOUT_MS = 300000; /** * The budget for one request, in milliseconds. * * @param params - LLM parameters, whose `requestTimeoutMs` wins when set * @param defaultMs - Budget to use when the caller named none * @returns The effective budget in milliseconds */ declare function resolveRequestTimeoutMs(params: Pick, defaultMs?: number): number; /** * Per-request options for an OpenAI- or Anthropic-style SDK call. * * A caller-supplied budget is a *ceiling*, not a per-attempt allowance, so * retries are disabled for it — three attempts at the requested budget would * spend three times what the caller agreed to. Returns undefined when the * caller named no budget, leaving the client's own defaults in charge. * * @param params - LLM parameters for the request * @returns SDK request options, or undefined to use the client defaults */ declare function buildSdkRequestOptions(params: Pick): { timeout: number; maxRetries: number; } | undefined; /** * Client-construction options for a provider that builds a fresh SDK client * per call, where "client default" and "this request" are the same thing. * * Always defined, unlike {@link buildSdkRequestOptions} — a per-call client has * no pre-existing defaults worth preserving, so it takes the resolved budget * either way and only forgoes retries when the caller set a ceiling. * * @param params - LLM parameters for the request * @param defaultMs - Budget to use when the caller named none * @returns `timeout` and `maxRetries` to spread into the SDK constructor */ declare function buildSdkClientOptions(params: Pick, defaultMs?: number): { timeout: number; maxRetries: number; }; /** * An abort signal that fires at the request's budget, for providers that call * `fetch` directly rather than through an SDK. * * Unlike the SDK options above this bounds the *whole* exchange, body included: * `fetch` resolves at the response headers, but the signal stays live while the * body streams. Pass a generous `defaultMs` on streaming paths, or compose with * the caller's own signal, so a long generation is not cut off mid-answer. * * @param params - LLM parameters for the request * @param defaultMs - Budget to use when the caller named none * @returns An AbortSignal that aborts once the budget elapses */ declare function buildRequestAbortSignal(params: Pick, defaultMs?: number): AbortSignal; export { DEFAULT_REQUEST_TIMEOUT_MS, OpenAICompatibleProvider, type OpenAICompatibleProviderConfig, type ProfileParamNormalizer, applyProfileParameters, buildRequestAbortSignal, buildSdkClientOptions, buildSdkRequestOptions, collapseLeadingSystemMessages, resolveRequestTimeoutMs };